airbyte-source-google-search-console 1.8.0__py3-none-any.whl → 1.9.1__py3-none-any.whl

{airbyte_source_google_search_console-1.8.0.dist-info → airbyte_source_google_search_console-1.9.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: airbyte-source-google-search-console
-Version: 1.8.0
+Version: 1.9.1
 Summary: Source implementation for Google Search Console.
 License: Elv2
 Author: Airbyte

{airbyte_source_google_search_console-1.8.0.dist-info → airbyte_source_google_search_console-1.9.1.dist-info}/RECORD

@@ -1,14 +1,13 @@
 source_google_search_console/__init__.py,sha256=HQCPu-CK7XmVDtP9rmTdB2XyraVCc6pv9pw38-O8y48,1191
-source_google_search_console/components.py,sha256=5o8kH2xwYUvk3yjnSd6okJVF2KBUqUIF4V97xRpPAyI,2803
+source_google_search_console/components.py,sha256=_6gl-xgwmvRgChEGNZhMsuYAcTw7gIf3yfe7IKvWkPk,6767
 source_google_search_console/config_migrations.py,sha256=Cl4SUdJpAf6wMM_vVhqjjU89NfUq9LIGJ9zNrWiBk-A,4235
 source_google_search_console/exceptions.py,sha256=iD3jYC4WxVCEKGsqQ7Vaj1tbjhJZ4S5mnSDnwFJdsIQ,1097
-source_google_search_console/manifest.yaml,sha256=3zquI6ykliHeQtD3DRgt0ajRvDCzlmJHqB6lFzOIMfQ,49048
+source_google_search_console/manifest.yaml,sha256=luNdDPiw_wOoZPFEHFiO7sSUH-AG_cpBiRhPses7vGQ,55028
 source_google_search_console/run.py,sha256=TBkPlseTERarkj6wL8AMEKgm5Xsb2drnltPVH6257-M,2195
 source_google_search_console/service_account_authenticator.py,sha256=pAWKAXfwfTY3xkXvQJH0EyFphFULdCIcC47YXYTO9X8,1307
-source_google_search_console/source.py,sha256=9sqb9b1xnIRH5BN_8BGvmmiLuCG-GkIHMCCi33C7fcE,7918
+source_google_search_console/source.py,sha256=TFuafbBg8Nlb-LsYwNSXH1Xcz9U0ELZRX0e3hjCKPzs,7669
 source_google_search_console/spec.json,sha256=HebTMHop7twcJH6jjgAccYEgg93bTLGsp6jVdMj1x0c,9409
-source_google_search_console/streams.py,sha256=dboFHRxX4aEV4IKm1ZDTKgoxkBuhUPXhhcJy9EUlIP0,14011
-airbyte_source_google_search_console-1.8.0.dist-info/METADATA,sha256=OcmJ9LRmcqFG70GsSejyv1KFoMjPzpxwQ4l-2U6q0WU,5621
-airbyte_source_google_search_console-1.8.0.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-airbyte_source_google_search_console-1.8.0.dist-info/entry_points.txt,sha256=DMcgc9bCX-Vt6hm_68pa77qS3eGdeMhg-UdlFc-XKUM,85
-airbyte_source_google_search_console-1.8.0.dist-info/RECORD,,
+airbyte_source_google_search_console-1.9.1.dist-info/METADATA,sha256=6dftcAmz795H8LN_VxuYk9xXCEN9uhVfoK5sRBHl-No,5621
+airbyte_source_google_search_console-1.9.1.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+airbyte_source_google_search_console-1.9.1.dist-info/entry_points.txt,sha256=DMcgc9bCX-Vt6hm_68pa77qS3eGdeMhg-UdlFc-XKUM,85
+airbyte_source_google_search_console-1.9.1.dist-info/RECORD,,

source_google_search_console/components.py

@@ -2,10 +2,13 @@
 # Copyright (c) 2025 Airbyte, Inc., all rights reserved.
 #
 
-from dataclasses import dataclass
-from typing import Any, Mapping, Optional
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Mapping, Optional
 
 from airbyte_cdk.sources.declarative.migrations.state_migration import StateMigration
+from airbyte_cdk.sources.declarative.schema import SchemaLoader
+from airbyte_cdk.sources.declarative.transformations import RecordTransformation
+from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
 
 
 @dataclass
@@ -59,7 +62,6 @@ class NestedSubstreamStateMigration(StateMigration):
         return len(stream_state) > 0 and "states" not in stream_state
 
     def migrate(self, stream_state: Mapping[str, Any]) -> Mapping[str, Any]:
-        global_state: Optional[Mapping[str, Any]] = None
         per_partition_state = []
         for site_url_key, search_type_state in stream_state.items():
             if site_url_key == "date":
@@ -79,3 +81,103 @@ class NestedSubstreamStateMigration(StateMigration):
             "use_global_cursor": False,
             "states": per_partition_state,
         }
+
+
+@dataclass
+class CustomReportExtractDimensionsFromKeys(RecordTransformation):
+    """
+    A record transformation that remaps each value in the keys array back to its associated
+    dimension. The reason this is a custom component is because we're unable to use list
+    comprehension and and enumerate() is not a valid function in our Jinja contact so can't
+    iterate over the dimensions defined in the config to create each field transformation on the
+    stream_template for each custom report.
+
+    If we were able to, the actual ComponentMappingDefinition would look like this:
+
+    type: ComponentMappingDefinition
+    field_path:
+      - transformations
+      - "1"
+      - fields
+    value: "{{ [{'path': [dimension], 'value': '{{ record['keys'][index]} for index, dimension in enumerate(components_values['dimensions'])] }}"
+
+    or
+
+    type: ComponentMappingDefinition
+    field_path:
+      - transformations
+      - "1"
+      - fields
+    value: >
+      {% for index, dimension in enumerate(components_values["dimensions"]) %}
+           - type: AddFields
+             fields:
+               - path: [ {{ dimension }} ]
+                 value: "{{ record['keys'][index] }}"
+      {% endfor %}
+    """
+
+    dimensions: List[str] = field(default_factory=lambda: [])
+
+    def transform(
+        self,
+        record: Dict[str, Any],
+        config: Optional[Config] = None,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+    ) -> None:
+        for dimension in self.dimensions:
+            record[dimension] = record["keys"].pop(0)
+
+        record.pop("keys")
+
+
+@dataclass
+class CustomReportSchemaLoader(SchemaLoader):
+    """
+    Custom schema loader is needed because Google Search Console's custom reports streams
+    because the schema is dependent on which dimensions are selected in the config. Right now,
+    only DynamicSchemaLoader which is based on the response from an API endpoint supports
+    remapping of types to Airbyte schema types. This CustomReportSchemaLoader functions
+    more like a static schema loader and so we must perform the remapping in this custom component.
+    """
+
+    DIMENSION_TO_PROPERTY_SCHEMA_MAP = {
+        "country": [{"country": {"type": ["null", "string"]}}],
+        "date": [{"date": {"type": ["null", "string"], "format": "date"}}],
+        "device": [{"device": {"type": ["null", "string"]}}],
+        "page": [{"page": {"type": ["null", "string"]}}],
+        "query": [{"query": {"type": ["null", "string"]}}],
+    }
+
+    dimensions: List[str]
+
+    def get_json_schema(self) -> Mapping[str, Any]:
+        schema: Mapping[str, Any] = {
+            "$schema": "https://json-schema.org/draft-07/schema#",
+            "type": ["null", "object"],
+            "additionalProperties": True,
+            "properties": {
+                # metrics
+                "clicks": {"type": ["null", "integer"]},
+                "ctr": {"type": ["null", "number"], "multipleOf": 1e-25},
+                "impressions": {"type": ["null", "integer"]},
+                "position": {"type": ["null", "number"], "multipleOf": 1e-25},
+                # default fields
+                "search_type": {"type": ["null", "string"]},
+                "site_url": {"type": ["null", "string"]},
+            },
+        }
+
+        # dimensions
+        dimension_properties = self._dimension_to_property_schema()
+        schema["properties"].update(dimension_properties)
+        return schema
+
+    def _dimension_to_property_schema(self) -> dict:
+        properties = {}
+        for dimension in sorted(self.dimensions):
+            fields = self.DIMENSION_TO_PROPERTY_SCHEMA_MAP[dimension]
+            for field in fields:
+                properties = {**properties, **field}
+        return properties

source_google_search_console/manifest.yaml

@@ -58,6 +58,19 @@ definitions:
           type: "{{ stream_partition.get('search_type') }}"
           aggregationType: "{{ 'auto' if config.get('always_use_aggregation_type_auto') else parameters.get('aggregationType') }}"
           dataState: "{{ config.get('data_state', 'final') }}"
+        error_handler:
+          type: DefaultErrorHandler
+          response_filters:
+            - type: HttpResponseFilter
+              action: RATE_LIMITED
+              error_message_contains: "Search Analytics QPS quota exceeded"
+            - type: HttpResponseFilter
+              action: FAIL
+              http_codes:
+                - 400
+              error_message: >-
+                Invalid aggregationType '{{ parameters.get('aggregationType') }}' used in the body of the API request. If you see this error, enable the
+                'always_use_aggregation_type_auto' config setting which will automatically use aggregationType=auto
       paginator:
         type: DefaultPaginator
         page_token_option:
@@ -605,8 +618,8 @@ definitions:
         request_headers:
           Content-Type: "application/json"
         request_body_json:
-          startDate: "{{ config.get('start_date') }}"
-          endDate: "{{ config.get('end_date') }}"
+          startDate: "{{ config.get('start_date', '2021-01-01') }}"
+          endDate: "{{ config.get('end_date', today_utc()) }}"
           dimensions: ["searchAppearance"]
           type: "{{ stream_partition.get('search_type') }}"
           aggregationType: auto
@@ -675,6 +688,9 @@ definitions:
         error_handler:
           type: DefaultErrorHandler
           response_filters:
+            - type: HttpResponseFilter
+              action: RATE_LIMITED
+              error_message_contains: "Search Analytics QPS quota exceeded"
             - type: HttpResponseFilter
               action: FAIL
               http_codes:
@@ -897,6 +913,135 @@ streams:
   - "#/definitions/search_analytics_keyword_site_report_by_page_stream"
   - "#/definitions/search_analytics_keyword_site_report_by_site_stream"
 
+dynamic_streams:
+  - type: DynamicDeclarativeStream
+    stream_template:
+      type: DeclarativeStream
+      name: search_analytics_by_custom_dimensions # This will be replaced by the name of the custom report
+      primary_key: # This will be replaced by the dimensions of the custom report
+        - site_url
+        - search_type
+      retriever:
+        type: SimpleRetriever
+        requester:
+          type: HttpRequester
+          url_base: https://www.googleapis.com/webmasters/v3
+          path: "/sites/{{  sanitize_url(stream_partition.get('site_url')) }}/searchAnalytics/query"
+          http_method: POST
+          authenticator: "#/definitions/selective_authenticator"
+          request_headers:
+            Content-Type: "application/json"
+          request_body_json:
+            startDate: "{{ stream_interval.get('start_time') }}"
+            endDate: "{{ stream_interval.get('end_time') }}"
+            dimensions: ["date", "country"] # This will be replaced by the dimensions of the custom report
+            type: "{{ stream_partition.get('search_type') }}"
+            aggregationType: auto
+            dataState: "{{ config.get('data_state', 'final') }}"
+        paginator:
+          type: DefaultPaginator
+          page_token_option:
+            type: RequestOption
+            field_name: startRow
+            inject_into: body_json
+          page_size_option:
+            type: RequestOption
+            field_name: rowLimit
+            inject_into: body_json
+          pagination_strategy:
+            type: OffsetIncrement
+            page_size: 25000
+            inject_on_first_request: true
+        record_selector:
+          type: RecordSelector
+          extractor:
+            type: DpathExtractor
+            field_path:
+              - rows
+        partition_router:
+          - type: ListPartitionRouter
+            values: "{{ config['site_urls'] }}"
+            cursor_field: site_url
+          - type: ListPartitionRouter
+            values:
+              - web
+              - news
+              - image
+              - video
+            cursor_field: search_type
+      incremental_sync:
+        type: DatetimeBasedCursor
+        cursor_field: date
+        cursor_datetime_formats:
+          - "%Y-%m-%d"
+        datetime_format: "%Y-%m-%d"
+        start_datetime:
+          type: MinMaxDatetime
+          datetime: "{{ config.get('start_date', '2021-01-01') }}"
+          datetime_format: "%Y-%m-%d"
+        end_datetime:
+          type: MinMaxDatetime
+          datetime: "{{ config.get('end_date', today_utc()) }}"
+          datetime_format: "%Y-%m-%d"
+        step: P3D
+        cursor_granularity: P1D
+      transformations:
+        - type: AddFields
+          fields:
+            - path:
+                - site_url
+              value: "{{ stream_partition['site_url'] }}"
+            - path:
+                - search_type
+              value: "{{ stream_partition['search_type'] }}"
+        - type: CustomTransformation
+          class_name: source_google_search_console.components.CustomReportExtractDimensionsFromKeys
+          dimensions: # This will be replaced by the dimensions of the custom report
+            - date
+            - country
+      schema_loader:
+        type: CustomSchemaLoader
+        class_name: source_google_search_console.components.CustomReportSchemaLoader
+        dimensions: [] # This will be replaced by the dimensions of the custom report
+      state_migrations:
+        - type: CustomStateMigration
+          class_name: source_google_search_console.components.NestedSubstreamStateMigration
+    components_resolver:
+      type: ConfigComponentsResolver
+      stream_config:
+        type: StreamConfig
+        configs_pointer:
+          - custom_reports_array
+      components_mapping:
+        - type: ComponentMappingDefinition
+          field_path:
+            # - "**" # is this needed
+            - name
+          value: "{{components_values['name']}}"
+        - type: ComponentMappingDefinition
+          field_path:
+            - primary_key
+          value: "{{ components_values['dimensions'] + (['date'] if 'date' not in components_values['dimensions'] else []) + ['site_url', 'search_type'] }}"
+        - type: ComponentMappingDefinition
+          field_path:
+            - retriever
+            - requester
+            - request_body_json
+            - dimensions
+          # `date` is a cursor field therefore should be a mandatory dimension if not already present
+          value: "{{ components_values['dimensions'] + (['date'] if 'date' not in components_values['dimensions'] else []) }}"
+        - type: ComponentMappingDefinition
+          field_path:
+            - transformations
+            - "1"
+            - dimensions
+          value: "{{ components_values['dimensions'] + (['date'] if 'date' not in components_values['dimensions'] else []) }}"
+        - type: ComponentMappingDefinition
+          field_path:
+            - schema_loader
+            - dimensions
+          value: "{{ components_values['dimensions'] + (['date'] if 'date' not in components_values['dimensions'] else []) }}"
+
 # Google Search Console has three layers of quotas that dictate rate limiting at the
 # user making requests, site being requested, and developer console key used.
 # https://developers.google.com/webmaster-tools/limits#qps-quota
@@ -905,13 +1050,20 @@ streams:
 # - Per Project Quota: 30,000,000 req/day (350 req/sec) / 40,000 req/min (60 req/sec)
 #
 # The most likely upper bound is based on the user quota since it is the lowest and the
-# same authenticated user account may hit multiple site urls. The default is set to 40
-# which equates to one request every 2 seconds which seems like a fair baseline.
-#
+# same authenticated user account may hit multiple site urls.
+api_budget:
+  type: HTTPAPIBudget
+  policies:
+    - type: MovingWindowCallRatePolicy
+      rates:
+        - limit: 1200
+          interval: PT1M
+      matchers: []
+
 concurrency_level:
   type: ConcurrencyLevel
-  default_concurrency: "{{ config.get('num_workers', 40) }}"
-  max_concurrency: 100
+  default_concurrency: "{{ config.get('num_workers', 3) }}"
+  max_concurrency: 50
 
 schemas:
   search_analytics_all_fields:

source_google_search_console/source.py

@@ -22,11 +22,16 @@ from source_google_search_console.exceptions import (
     UnidentifiedError,
 )
 from source_google_search_console.service_account_authenticator import ServiceAccountAuthenticator
-from source_google_search_console.streams import (
-    SearchAnalyticsByCustomDimensions,
-)
 
 
+DIMENSION_TO_PROPERTY_SCHEMA_MAP = {
+    "country": [{"country": {"type": ["null", "string"]}}],
+    "date": [{"date": {"type": ["null", "string"], "format": "date"}}],
+    "device": [{"device": {"type": ["null", "string"]}}],
+    "page": [{"page": {"type": ["null", "string"]}}],
+    "query": [{"query": {"type": ["null", "string"]}}],
+}
+
 custom_reports_schema = {
     "type": "array",
     "items": {
@@ -93,7 +98,7 @@ class SourceGoogleSearchConsole(YamlDeclarativeSource):
             jsonschema.validate(config["custom_reports_array"], custom_reports_schema)
             for report in config["custom_reports_array"]:
                 for dimension in report["dimensions"]:
-                    if dimension not in SearchAnalyticsByCustomDimensions.DIMENSION_TO_PROPERTY_SCHEMA_MAP:
+                    if dimension not in DIMENSION_TO_PROPERTY_SCHEMA_MAP:
                         message = f"dimension: '{dimension}' not found"
                         raise AirbyteTracedException(message=message, internal_message=message, failure_type=FailureType.config_error)
         return config
@@ -137,17 +142,7 @@ class SourceGoogleSearchConsole(YamlDeclarativeSource):
         config = self._validate_and_transform(config)
         stream_config = self.get_stream_kwargs(config)
 
-        streams = super().streams(config=config)
-
-        streams = streams + self.get_custom_reports(config=config, stream_config=stream_config)
-
-        return streams
-
-    def get_custom_reports(self, config: Mapping[str, Any], stream_config: Mapping[str, Any]) -> List[Optional[Stream]]:
-        return [
-            type(report["name"], (SearchAnalyticsByCustomDimensions,), {})(dimensions=report["dimensions"], **stream_config)
-            for report in config.get("custom_reports_array", [])
-        ]
+        return super().streams(config=config)
 
     def get_stream_kwargs(self, config: Mapping[str, Any]) -> Mapping[str, Any]:
         return {

source_google_search_console/streams.py

@@ -1,347 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-from abc import ABC
-from enum import Enum
-from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Union
-from urllib.parse import quote_plus, unquote_plus
-
-import pendulum
-import requests
-from requests.auth import AuthBase
-
-from airbyte_cdk.models import SyncMode
-from airbyte_cdk.sources.streams import CheckpointMixin
-from airbyte_cdk.sources.streams.http import HttpStream
-
-
-BASE_URL = "https://www.googleapis.com/webmasters/v3/"
-ROW_LIMIT = 25000
-
-
-class QueryAggregationType(Enum):
-    auto = "auto"
-    by_page = "byPage"
-    by_property = "byProperty"
-
-
-class GoogleSearchConsole(HttpStream, ABC):
-    url_base = BASE_URL
-    data_field = ""
-    raise_on_http_errors = True
-
-    def __init__(
-        self,
-        authenticator: AuthBase,
-        site_urls: list,
-        start_date: str,
-        end_date: str,
-        data_state: str = "final",
-    ):
-        super().__init__(authenticator=authenticator)
-        self._site_urls = self.sanitize_urls_list(site_urls)
-        self._start_date = start_date
-        self._end_date = end_date
-        self._data_state = data_state
-
-    @staticmethod
-    def sanitize_urls_list(site_urls: list) -> List[str]:
-        return list(map(quote_plus, site_urls))
-
-    def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]:
-        return None
-
-    def stream_slices(
-        self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None
-    ) -> Iterable[Optional[Mapping[str, Any]]]:
-        for site_url in self._site_urls:
-            yield {"site_url": site_url}
-
-    def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
-        if not self.data_field:
-            yield response.json()
-
-        else:
-            records = response.json().get(self.data_field) or []
-            for record in records:
-                yield record
-
-    def should_retry(self, response: requests.Response) -> bool:
-        response_json = response.json()
-        if "error" in response_json:
-            error = response_json.get("error", {})
-            # handle the `HTTP-403` - insufficient permissions
-            if error.get("code", 0) == 403:
-                self.logger.error(f"Stream {self.name}. {error.get('message')}. Skipping.")
-                setattr(self, "raise_on_http_errors", False)
-                return False
-            # handle the `HTTP-400` - Bad query params with `aggregationType`
-            if error.get("code", 0) == 400:
-                self.logger.error(f"Stream `{self.name}`. {error.get('message')}. Trying with `aggregationType = auto` instead.")
-                self.aggregation_type = QueryAggregationType.auto
-                setattr(self, "raise_on_http_errors", False)
-        return response.status_code == 429 or 500 <= response.status_code < 600
-
-
-class SearchAnalytics(GoogleSearchConsole, CheckpointMixin, ABC):
-    """
-    API docs: https://developers.google.com/webmaster-tools/search-console-api-original/v3/searchanalytics
-    """
-
-    data_field = "rows"
-    aggregation_type = QueryAggregationType.auto
-    start_row = 0
-    dimensions = []
-    search_types = ["web", "news", "image", "video"]
-    range_of_days = 3
-
-    def __init__(self, authenticator: AuthBase, site_urls: list, start_date: str, end_date: str, data_state: str = "final", **kwargs):
-        super().__init__(authenticator=authenticator, site_urls=site_urls, start_date=start_date, end_date=end_date, data_state=data_state)
-        self._state = {}
-
-    def path(
-        self,
-        stream_state: Mapping[str, Any] = None,
-        stream_slice: Mapping[str, Any] = None,
-        next_page_token: Mapping[str, Any] = None,
-    ) -> str:
-        return f"sites/{stream_slice.get('site_url')}/searchAnalytics/query"
-
-    @property
-    def cursor_field(self) -> Union[str, List[str]]:
-        return "date"
-
-    @property
-    def http_method(self) -> str:
-        return "POST"
-
-    @property
-    def state(self) -> MutableMapping[str, Any]:
-        return self._state
-
-    @state.setter
-    def state(self, value: MutableMapping[str, Any]):
-        self._state = value
-
-    def stream_slices(
-        self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None
-    ) -> Iterable[Optional[Mapping[str, Any]]]:
-        """
-        The `stream_slices` implements iterator functionality for `site_urls` and `searchType`. The user can pass many `site_url`,
-        and we have to process all of them, we can also pass the` searchType` parameter in the `request body` to get data using some`
-        searchType` value from [` web`, `news `,` image`, `video`,  `discover`, `googleNews`].
-        It's just a double nested loop with a yield statement.
-        """
-
-        for site_url in self._site_urls:
-            for search_type in self.search_types:
-                start_date = self._get_start_date(stream_state, site_url, search_type)
-                end_date = self._get_end_date()
-
-                if start_date > end_date:
-                    start_date = end_date
-
-                next_start = start_date
-                period = pendulum.Duration(days=self.range_of_days - 1)
-                while next_start <= end_date:
-                    next_end = min(next_start + period, end_date)
-                    yield {
-                        "site_url": site_url,
-                        "search_type": search_type,
-                        "start_date": next_start.to_date_string(),
-                        "end_date": next_end.to_date_string(),
-                        "data_state": self._data_state,
-                    }
-                    # add 1 day for the next slice's start date not to duplicate data from previous slice's end date.
-                    next_start = next_end + pendulum.Duration(days=1)
-
-    def next_page_token(self, response: requests.Response) -> Optional[bool]:
-        """
-        The `next_page_token` implements pagination functionality. This method gets the response
-        and compares the number of records with the constant `ROW_LIMITS` (maximum value 25000),
-        and if they are equal, this means that we get the end of the` Page`, and we need to go further,
-        for this we simply increase the `startRow` parameter in request body by `ROW_LIMIT` value.
-        """
-
-        if len(response.json().get(self.data_field, [])) == ROW_LIMIT:
-            self.start_row += ROW_LIMIT
-            return True
-
-        self.start_row = 0
-
-    def request_headers(self, **kwargs) -> Mapping[str, Any]:
-        return {"Content-Type": "application/json"}
-
-    def request_body_json(
-        self,
-        stream_state: Mapping[str, Any] = None,
-        stream_slice: Mapping[str, Any] = None,
-        next_page_token: Mapping[str, Any] = None,
-    ) -> Optional[Union[Dict[str, Any], str]]:
-        """
-        Here is a description of the parameters and implementations of the request body:
-        1. The `startDate` is retrieved from the `_get_start_date`,
-        if` SyncMode = full_refresh` just use `start_date` from configuration, otherwise use `get_update_state`.
-        2. The `endDate` is retrieved from the `config.json`.
-        3. The `sizes` parameter is used to group the result by some dimension.
-        The following dimensions are available: `date`, `country`, `page`, `device`, `query`.
-        4. For the `type` check the paragraph stream_slices method.
-         Filter results to the following type ["web", "news", "image", "video", "discover", "googleNews"]
-        5. For the `startRow` and `rowLimit` check next_page_token method.
-        """
-
-        data = {
-            "startDate": stream_slice["start_date"],
-            "endDate": stream_slice["end_date"],
-            "dimensions": self.dimensions,
-            "type": stream_slice.get("search_type"),
-            "aggregationType": self.aggregation_type.value,
-            "startRow": self.start_row,
-            "rowLimit": ROW_LIMIT,
-            "dataState": stream_slice.get("data_state"),
-        }
-
-        return data
-
-    def _get_end_date(self) -> pendulum.date:
-        end_date = pendulum.parse(self._end_date).date()
-        # limit `end_date` value with current date
-        return min(end_date, pendulum.now().date())
-
-    def _get_start_date(self, stream_state: Mapping[str, Any] = None, site_url: str = None, search_type: str = None) -> pendulum.date:
-        start_date = pendulum.parse(self._start_date)
-
-        if start_date and stream_state:
-            if stream_state.get(unquote_plus(site_url), {}).get(search_type):
-                stream_state_value = stream_state.get(unquote_plus(site_url), {}).get(search_type)
-
-                start_date = max(
-                    pendulum.parse(stream_state_value[self.cursor_field]),
-                    start_date,
-                )
-
-        return start_date.date()
-
-    def parse_response(
-        self,
-        response: requests.Response,
-        stream_state: Mapping[str, Any],
-        stream_slice: Mapping[str, Any] = None,
-        next_page_token: Mapping[str, Any] = None,
-    ) -> Iterable[Mapping]:
-        records = response.json().get(self.data_field) or []
-
-        for record in records:
-            record["site_url"] = unquote_plus(stream_slice.get("site_url"))
-            record["search_type"] = stream_slice.get("search_type")
-
-            for dimension in self.dimensions:
-                record[dimension] = record["keys"].pop(0)
-
-            # remove unnecessary empty field
-            record.pop("keys")
-
-            yield record
-
-    def _get_updated_state(
-        self,
-        current_stream_state: MutableMapping[str, Any],
-        latest_record: Mapping[str, Any],
-    ) -> Mapping[str, Any]:
-        """
-        With the existing nested loop implementation, we have to store a `cursor_field` for each `site_url`
-        and `searchType`. This functionality is placed in `get_update_state`.
-
-        {
-          "stream": {
-            "https://domain1.com": {
-              "web": {"date": "2022-01-03"},
-              "news": {"date": "2022-01-03"},
-              "image": {"date": "2022-01-03"},
-              "video": {"date": "2022-01-03"}
-            },
-            "https://domain2.com": {
-              "web": {"date": "2022-01-03"},
-              "news": {"date": "2022-01-03"},
-              "image": {"date": "2022-01-03"},
-              "video": {"date": "2022-01-03"}
-            },
-            "date": "2022-01-03",
-          }
-        }
-        """
-
-        latest_benchmark = latest_record.get(self.cursor_field)
-
-        site_url = latest_record.get("site_url")
-        search_type = latest_record.get("search_type")
-
-        value = current_stream_state.get(site_url, {}).get(search_type, {}).get(self.cursor_field)
-        if value:
-            latest_benchmark = max(latest_benchmark, value)
-        current_stream_state.setdefault(site_url, {}).setdefault(search_type, {})[self.cursor_field] = latest_benchmark
-
-        # we need to get the max date over all searchTypes but the current acceptance test YAML format doesn't
-        # support that
-        current_stream_state[self.cursor_field] = current_stream_state[site_url][search_type][self.cursor_field]
-
-        return current_stream_state
-
-    def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]:
-        for record in super().read_records(**kwargs):
-            self.state = self._get_updated_state(self.state, record)
-            yield record
-
-
-class SearchAnalyticsByCustomDimensions(SearchAnalytics):
-    # `date` is a cursor field therefore should be mandatory
-    DEFAULT_DIMENSIONS = ["date"]
-    DIMENSION_TO_PROPERTY_SCHEMA_MAP = {
-        "country": [{"country": {"type": ["null", "string"]}}],
-        "date": [{"date": {"type": ["null", "string"], "format": "date"}}],
-        "device": [{"device": {"type": ["null", "string"]}}],
-        "page": [{"page": {"type": ["null", "string"]}}],
-        "query": [{"query": {"type": ["null", "string"]}}],
-    }
-
-    primary_key = None
-
-    def __init__(self, dimensions: List[str], *args, **kwargs):
-        super(SearchAnalyticsByCustomDimensions, self).__init__(*args, **kwargs)
-        self.dimensions = dimensions + [dimension for dimension in self.DEFAULT_DIMENSIONS if dimension not in dimensions]
-        # Assign the dimensions as PK for the custom report stream.
-        # Site URL and Search Type are included in the API call thus affect the resulting data.
-        # `site_url` is a required URL param for making API calls;
-        # `search_type` remains a query param for historical reasons, we do not want to remove it to not break existing connections.
-        self.primary_key = self.dimensions + ["site_url", "search_type"]
-
-    def get_json_schema(self) -> Mapping[str, Any]:
-        schema: Mapping[str, Any] = {
-            "$schema": "https://json-schema.org/draft-07/schema#",
-            "type": ["null", "object"],
-            "additionalProperties": True,
-            "properties": {
-                # metrics
-                "clicks": {"type": ["null", "integer"]},
-                "ctr": {"type": ["null", "number"], "multipleOf": 1e-25},
-                "impressions": {"type": ["null", "integer"]},
-                "position": {"type": ["null", "number"], "multipleOf": 1e-25},
-                # default fields
-                "search_type": {"type": ["null", "string"]},
-                "site_url": {"type": ["null", "string"]},
-            },
-        }
-
-        # dimensions
-        dimension_properties = self.dimension_to_property_schema()
-        schema["properties"].update(dimension_properties)
-        return schema
-
-    def dimension_to_property_schema(self) -> dict:
-        properties = {}
-        for dimension in sorted(self.dimensions):
-            fields = self.DIMENSION_TO_PROPERTY_SCHEMA_MAP[dimension]
-            for field in fields:
-                properties = {**properties, **field}
-        return properties