airbyte-source-google-search-console 1.8.0__tar.gz → 1.9.0__tar.gz

{airbyte_source_google_search_console-1.8.0 → airbyte_source_google_search_console-1.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: airbyte-source-google-search-console
-Version: 1.8.0
+Version: 1.9.0
 Summary: Source implementation for Google Search Console.
 License: Elv2
 Author: Airbyte

{airbyte_source_google_search_console-1.8.0 → airbyte_source_google_search_console-1.9.0}/pyproject.toml

@@ -5,7 +5,7 @@ requires = [
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "1.8.0"
+version = "1.9.0"
 name = "airbyte-source-google-search-console"
 description = "Source implementation for Google Search Console."
 authors = [

airbyte_source_google_search_console-1.9.0/source_google_search_console/components.py

@@ -0,0 +1,183 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Mapping, Optional
+
+from airbyte_cdk.sources.declarative.migrations.state_migration import StateMigration
+from airbyte_cdk.sources.declarative.schema import SchemaLoader
+from airbyte_cdk.sources.declarative.transformations import RecordTransformation
+from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
+
+
+@dataclass
+class NestedSubstreamStateMigration(StateMigration):
+    """
+    We require a custom state migration because SearchAnalytics streams contain two nested levels of
+    substreams. The existing LegacyToPerPartitionStateMigration only handles one level.
+
+    Legacy state format is as follows:
+    {
+        "date": "2025-05-28",
+        "https://www.example.com/": {
+            "web": {
+                "date": "2025-05-25"
+            },
+            "news": {
+                "date": "2023-05-22"
+            }
+        }
+    }
+
+    The resulting migrated per-partition state is:
+    {
+        "use_global_cursor": false,
+        "states": [
+        {
+            "partition": {
+                "search_type": "web",
+                "site_url": "https://www.example.com/"
+            },
+            "cursor": {
+                "date": "2025-05-25"
+            }
+        },
+        {
+            "partition": {
+                "search_type": "news",
+                "site_url": "https://www.example.com/"
+            },
+            "cursor": {
+                "date": "2023-05-22"
+            }
+        }],
+        "state": {
+            "date": "2025-05-25"
+        }
+    }
+    """
+
+    def should_migrate(self, stream_state: Mapping[str, Any]) -> bool:
+        return len(stream_state) > 0 and "states" not in stream_state
+
+    def migrate(self, stream_state: Mapping[str, Any]) -> Mapping[str, Any]:
+        per_partition_state = []
+        for site_url_key, search_type_state in stream_state.items():
+            if site_url_key == "date":
+                # The legacy state also contains a global cursor value under the `date` key which equates
+                # to global state.
+                #
+                # However, the Python implementation does not appear to be implemented
+                # correctly and simply saves the state of the last seen partition. Since I don't trust the
+                # legacy value and in the current implementation global state is applied to partitions
+                # without an existing value, I'm making a conscious choice to not migrate the global value.
+                continue
+            else:
+                site_url = site_url_key
+                for search_type_key, cursor in search_type_state.items():
+                    per_partition_state.append({"partition": {"site_url": site_url, "search_type": search_type_key}, "cursor": cursor})
+        return {
+            "use_global_cursor": False,
+            "states": per_partition_state,
+        }
+
+
+@dataclass
+class CustomReportExtractDimensionsFromKeys(RecordTransformation):
+    """
+    A record transformation that remaps each value in the keys array back to its associated
+    dimension. The reason this is a custom component is because we're unable to use list
+    comprehension and and enumerate() is not a valid function in our Jinja contact so can't
+    iterate over the dimensions defined in the config to create each field transformation on the
+    stream_template for each custom report.
+
+    If we were able to, the actual ComponentMappingDefinition would look like this:
+
+    type: ComponentMappingDefinition
+    field_path:
+      - transformations
+      - "1"
+      - fields
+    value: "{{ [{'path': [dimension], 'value': '{{ record['keys'][index]} for index, dimension in enumerate(components_values['dimensions'])] }}"
+
+    or
+
+    type: ComponentMappingDefinition
+    field_path:
+      - transformations
+      - "1"
+      - fields
+    value: >
+      {% for index, dimension in enumerate(components_values["dimensions"]) %}
+           - type: AddFields
+             fields:
+               - path: [ {{ dimension }} ]
+                 value: "{{ record['keys'][index] }}"
+      {% endfor %}
+    """
+
+    dimensions: List[str] = field(default_factory=lambda: [])
+
+    def transform(
+        self,
+        record: Dict[str, Any],
+        config: Optional[Config] = None,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+    ) -> None:
+        for dimension in self.dimensions:
+            record[dimension] = record["keys"].pop(0)
+
+        record.pop("keys")
+
+
+@dataclass
+class CustomReportSchemaLoader(SchemaLoader):
+    """
+    Custom schema loader is needed because Google Search Console's custom reports streams
+    because the schema is dependent on which dimensions are selected in the config. Right now,
+    only DynamicSchemaLoader which is based on the response from an API endpoint supports
+    remapping of types to Airbyte schema types. This CustomReportSchemaLoader functions
+    more like a static schema loader and so we must perform the remapping in this custom component.
+    """
+
+    DIMENSION_TO_PROPERTY_SCHEMA_MAP = {
+        "country": [{"country": {"type": ["null", "string"]}}],
+        "date": [{"date": {"type": ["null", "string"], "format": "date"}}],
+        "device": [{"device": {"type": ["null", "string"]}}],
+        "page": [{"page": {"type": ["null", "string"]}}],
+        "query": [{"query": {"type": ["null", "string"]}}],
+    }
+
+    dimensions: List[str]
+
+    def get_json_schema(self) -> Mapping[str, Any]:
+        schema: Mapping[str, Any] = {
+            "$schema": "https://json-schema.org/draft-07/schema#",
+            "type": ["null", "object"],
+            "additionalProperties": True,
+            "properties": {
+                # metrics
+                "clicks": {"type": ["null", "integer"]},
+                "ctr": {"type": ["null", "number"], "multipleOf": 1e-25},
+                "impressions": {"type": ["null", "integer"]},
+                "position": {"type": ["null", "number"], "multipleOf": 1e-25},
+                # default fields
+                "search_type": {"type": ["null", "string"]},
+                "site_url": {"type": ["null", "string"]},
+            },
+        }
+
+        # dimensions
+        dimension_properties = self._dimension_to_property_schema()
+        schema["properties"].update(dimension_properties)
+        return schema
+
+    def _dimension_to_property_schema(self) -> dict:
+        properties = {}
+        for dimension in sorted(self.dimensions):
+            fields = self.DIMENSION_TO_PROPERTY_SCHEMA_MAP[dimension]
+            for field in fields:
+                properties = {**properties, **field}
+        return properties

{airbyte_source_google_search_console-1.8.0 → airbyte_source_google_search_console-1.9.0}/source_google_search_console/manifest.yaml

@@ -897,6 +897,135 @@ streams:
   - "#/definitions/search_analytics_keyword_site_report_by_page_stream"
   - "#/definitions/search_analytics_keyword_site_report_by_site_stream"
 
+dynamic_streams:
+  - type: DynamicDeclarativeStream
+    stream_template:
+      type: DeclarativeStream
+      name: search_analytics_by_custom_dimensions # This will be replaced by the name of the custom report
+      primary_key: # This will be replaced by the dimensions of the custom report
+        - site_url
+        - search_type
+      retriever:
+        type: SimpleRetriever
+        requester:
+          type: HttpRequester
+          url_base: https://www.googleapis.com/webmasters/v3
+          path: "/sites/{{  sanitize_url(stream_partition.get('site_url')) }}/searchAnalytics/query"
+          http_method: POST
+          authenticator: "#/definitions/selective_authenticator"
+          request_headers:
+            Content-Type: "application/json"
+          request_body_json:
+            startDate: "{{ stream_interval.get('start_time') }}"
+            endDate: "{{ stream_interval.get('end_time') }}"
+            dimensions: ["date", "country"] # This will be replaced by the dimensions of the custom report
+            type: "{{ stream_partition.get('search_type') }}"
+            aggregationType: auto
+            dataState: "{{ config.get('data_state', 'final') }}"
+        paginator:
+          type: DefaultPaginator
+          page_token_option:
+            type: RequestOption
+            field_name: startRow
+            inject_into: body_json
+          page_size_option:
+            type: RequestOption
+            field_name: rowLimit
+            inject_into: body_json
+          pagination_strategy:
+            type: OffsetIncrement
+            page_size: 25000
+            inject_on_first_request: true
+        record_selector:
+          type: RecordSelector
+          extractor:
+            type: DpathExtractor
+            field_path:
+              - rows
+        partition_router:
+          - type: ListPartitionRouter
+            values: "{{ config['site_urls'] }}"
+            cursor_field: site_url
+          - type: ListPartitionRouter
+            values:
+              - web
+              - news
+              - image
+              - video
+            cursor_field: search_type
+      incremental_sync:
+        type: DatetimeBasedCursor
+        cursor_field: date
+        cursor_datetime_formats:
+          - "%Y-%m-%d"
+        datetime_format: "%Y-%m-%d"
+        start_datetime:
+          type: MinMaxDatetime
+          datetime: "{{ config.get('start_date', '2021-01-01') }}"
+          datetime_format: "%Y-%m-%d"
+        end_datetime:
+          type: MinMaxDatetime
+          datetime: "{{ config.get('end_date', today_utc()) }}"
+          datetime_format: "%Y-%m-%d"
+        step: P3D
+        cursor_granularity: P1D
+      transformations:
+        - type: AddFields
+          fields:
+            - path:
+                - site_url
+              value: "{{ stream_partition['site_url'] }}"
+            - path:
+                - search_type
+              value: "{{ stream_partition['search_type'] }}"
+        - type: CustomTransformation
+          class_name: source_google_search_console.components.CustomReportExtractDimensionsFromKeys
+          dimensions: # This will be replaced by the dimensions of the custom report
+            - date
+            - country
+      schema_loader:
+        type: CustomSchemaLoader
+        class_name: source_google_search_console.components.CustomReportSchemaLoader
+        dimensions: [] # This will be replaced by the dimensions of the custom report
+      state_migrations:
+        - type: CustomStateMigration
+          class_name: source_google_search_console.components.NestedSubstreamStateMigration
+    components_resolver:
+      type: ConfigComponentsResolver
+      stream_config:
+        type: StreamConfig
+        configs_pointer:
+          - custom_reports_array
+      components_mapping:
+        - type: ComponentMappingDefinition
+          field_path:
+            # - "**" # is this needed
+            - name
+          value: "{{components_values['name']}}"
+        - type: ComponentMappingDefinition
+          field_path:
+            - primary_key
+          value: "{{ components_values['dimensions'] + (['date'] if 'date' not in components_values['dimensions'] else []) + ['site_url', 'search_type'] }}"
+        - type: ComponentMappingDefinition
+          field_path:
+            - retriever
+            - requester
+            - request_body_json
+            - dimensions
+          # `date` is a cursor field therefore should be a mandatory dimension if not already present
+          value: "{{ components_values['dimensions'] + (['date'] if 'date' not in components_values['dimensions'] else []) }}"
+        - type: ComponentMappingDefinition
+          field_path:
+            - transformations
+            - "1"
+            - dimensions
+          value: "{{ components_values['dimensions'] + (['date'] if 'date' not in components_values['dimensions'] else []) }}"
+        - type: ComponentMappingDefinition
+          field_path:
+            - schema_loader
+            - dimensions
+          value: "{{ components_values['dimensions'] + (['date'] if 'date' not in components_values['dimensions'] else []) }}"
+
 # Google Search Console has three layers of quotas that dictate rate limiting at the
 # user making requests, site being requested, and developer console key used.
 # https://developers.google.com/webmaster-tools/limits#qps-quota

{airbyte_source_google_search_console-1.8.0 → airbyte_source_google_search_console-1.9.0}/source_google_search_console/source.py

@@ -22,11 +22,16 @@ from source_google_search_console.exceptions import (
     UnidentifiedError,
 )
 from source_google_search_console.service_account_authenticator import ServiceAccountAuthenticator
-from source_google_search_console.streams import (
-    SearchAnalyticsByCustomDimensions,
-)
 
 
+DIMENSION_TO_PROPERTY_SCHEMA_MAP = {
+    "country": [{"country": {"type": ["null", "string"]}}],
+    "date": [{"date": {"type": ["null", "string"], "format": "date"}}],
+    "device": [{"device": {"type": ["null", "string"]}}],
+    "page": [{"page": {"type": ["null", "string"]}}],
+    "query": [{"query": {"type": ["null", "string"]}}],
+}
+
 custom_reports_schema = {
     "type": "array",
     "items": {
@@ -93,7 +98,7 @@ class SourceGoogleSearchConsole(YamlDeclarativeSource):
             jsonschema.validate(config["custom_reports_array"], custom_reports_schema)
             for report in config["custom_reports_array"]:
                 for dimension in report["dimensions"]:
-                    if dimension not in SearchAnalyticsByCustomDimensions.DIMENSION_TO_PROPERTY_SCHEMA_MAP:
+                    if dimension not in DIMENSION_TO_PROPERTY_SCHEMA_MAP:
                         message = f"dimension: '{dimension}' not found"
                         raise AirbyteTracedException(message=message, internal_message=message, failure_type=FailureType.config_error)
         return config
@@ -137,17 +142,7 @@ class SourceGoogleSearchConsole(YamlDeclarativeSource):
         config = self._validate_and_transform(config)
         stream_config = self.get_stream_kwargs(config)
 
-        streams = super().streams(config=config)
-
-        streams = streams + self.get_custom_reports(config=config, stream_config=stream_config)
-
-        return streams
-
-    def get_custom_reports(self, config: Mapping[str, Any], stream_config: Mapping[str, Any]) -> List[Optional[Stream]]:
-        return [
-            type(report["name"], (SearchAnalyticsByCustomDimensions,), {})(dimensions=report["dimensions"], **stream_config)
-            for report in config.get("custom_reports_array", [])
-        ]
+        return super().streams(config=config)
 
     def get_stream_kwargs(self, config: Mapping[str, Any]) -> Mapping[str, Any]:
         return {

airbyte_source_google_search_console-1.8.0/source_google_search_console/components.py

@@ -1,81 +0,0 @@
-#
-# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
-#
-
-from dataclasses import dataclass
-from typing import Any, Mapping, Optional
-
-from airbyte_cdk.sources.declarative.migrations.state_migration import StateMigration
-
-
-@dataclass
-class NestedSubstreamStateMigration(StateMigration):
-    """
-    We require a custom state migration because SearchAnalytics streams contain two nested levels of
-    substreams. The existing LegacyToPerPartitionStateMigration only handles one level.
-
-    Legacy state format is as follows:
-    {
-        "date": "2025-05-28",
-        "https://www.example.com/": {
-            "web": {
-                "date": "2025-05-25"
-            },
-            "news": {
-                "date": "2023-05-22"
-            }
-        }
-    }
-
-    The resulting migrated per-partition state is:
-    {
-        "use_global_cursor": false,
-        "states": [
-        {
-            "partition": {
-                "search_type": "web",
-                "site_url": "https://www.example.com/"
-            },
-            "cursor": {
-                "date": "2025-05-25"
-            }
-        },
-        {
-            "partition": {
-                "search_type": "news",
-                "site_url": "https://www.example.com/"
-            },
-            "cursor": {
-                "date": "2023-05-22"
-            }
-        }],
-        "state": {
-            "date": "2025-05-25"
-        }
-    }
-    """
-
-    def should_migrate(self, stream_state: Mapping[str, Any]) -> bool:
-        return len(stream_state) > 0 and "states" not in stream_state
-
-    def migrate(self, stream_state: Mapping[str, Any]) -> Mapping[str, Any]:
-        global_state: Optional[Mapping[str, Any]] = None
-        per_partition_state = []
-        for site_url_key, search_type_state in stream_state.items():
-            if site_url_key == "date":
-                # The legacy state also contains a global cursor value under the `date` key which equates
-                # to global state.
-                #
-                # However, the Python implementation does not appear to be implemented
-                # correctly and simply saves the state of the last seen partition. Since I don't trust the
-                # legacy value and in the current implementation global state is applied to partitions
-                # without an existing value, I'm making a conscious choice to not migrate the global value.
-                continue
-            else:
-                site_url = site_url_key
-                for search_type_key, cursor in search_type_state.items():
-                    per_partition_state.append({"partition": {"site_url": site_url, "search_type": search_type_key}, "cursor": cursor})
-        return {
-            "use_global_cursor": False,
-            "states": per_partition_state,
-        }

airbyte_source_google_search_console-1.8.0/source_google_search_console/streams.py

@@ -1,347 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-from abc import ABC
-from enum import Enum
-from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Union
-from urllib.parse import quote_plus, unquote_plus
-
-import pendulum
-import requests
-from requests.auth import AuthBase
-
-from airbyte_cdk.models import SyncMode
-from airbyte_cdk.sources.streams import CheckpointMixin
-from airbyte_cdk.sources.streams.http import HttpStream
-
-
-BASE_URL = "https://www.googleapis.com/webmasters/v3/"
-ROW_LIMIT = 25000
-
-
-class QueryAggregationType(Enum):
-    auto = "auto"
-    by_page = "byPage"
-    by_property = "byProperty"
-
-
-class GoogleSearchConsole(HttpStream, ABC):
-    url_base = BASE_URL
-    data_field = ""
-    raise_on_http_errors = True
-
-    def __init__(
-        self,
-        authenticator: AuthBase,
-        site_urls: list,
-        start_date: str,
-        end_date: str,
-        data_state: str = "final",
-    ):
-        super().__init__(authenticator=authenticator)
-        self._site_urls = self.sanitize_urls_list(site_urls)
-        self._start_date = start_date
-        self._end_date = end_date
-        self._data_state = data_state
-
-    @staticmethod
-    def sanitize_urls_list(site_urls: list) -> List[str]:
-        return list(map(quote_plus, site_urls))
-
-    def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]:
-        return None
-
-    def stream_slices(
-        self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None
-    ) -> Iterable[Optional[Mapping[str, Any]]]:
-        for site_url in self._site_urls:
-            yield {"site_url": site_url}
-
-    def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
-        if not self.data_field:
-            yield response.json()
-
-        else:
-            records = response.json().get(self.data_field) or []
-            for record in records:
-                yield record
-
-    def should_retry(self, response: requests.Response) -> bool:
-        response_json = response.json()
-        if "error" in response_json:
-            error = response_json.get("error", {})
-            # handle the `HTTP-403` - insufficient permissions
-            if error.get("code", 0) == 403:
-                self.logger.error(f"Stream {self.name}. {error.get('message')}. Skipping.")
-                setattr(self, "raise_on_http_errors", False)
-                return False
-            # handle the `HTTP-400` - Bad query params with `aggregationType`
-            if error.get("code", 0) == 400:
-                self.logger.error(f"Stream `{self.name}`. {error.get('message')}. Trying with `aggregationType = auto` instead.")
-                self.aggregation_type = QueryAggregationType.auto
-                setattr(self, "raise_on_http_errors", False)
-        return response.status_code == 429 or 500 <= response.status_code < 600
-
-
-class SearchAnalytics(GoogleSearchConsole, CheckpointMixin, ABC):
-    """
-    API docs: https://developers.google.com/webmaster-tools/search-console-api-original/v3/searchanalytics
-    """
-
-    data_field = "rows"
-    aggregation_type = QueryAggregationType.auto
-    start_row = 0
-    dimensions = []
-    search_types = ["web", "news", "image", "video"]
-    range_of_days = 3
-
-    def __init__(self, authenticator: AuthBase, site_urls: list, start_date: str, end_date: str, data_state: str = "final", **kwargs):
-        super().__init__(authenticator=authenticator, site_urls=site_urls, start_date=start_date, end_date=end_date, data_state=data_state)
-        self._state = {}
-
-    def path(
-        self,
-        stream_state: Mapping[str, Any] = None,
-        stream_slice: Mapping[str, Any] = None,
-        next_page_token: Mapping[str, Any] = None,
-    ) -> str:
-        return f"sites/{stream_slice.get('site_url')}/searchAnalytics/query"
-
-    @property
-    def cursor_field(self) -> Union[str, List[str]]:
-        return "date"
-
-    @property
-    def http_method(self) -> str:
-        return "POST"
-
-    @property
-    def state(self) -> MutableMapping[str, Any]:
-        return self._state
-
-    @state.setter
-    def state(self, value: MutableMapping[str, Any]):
-        self._state = value
-
-    def stream_slices(
-        self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None
-    ) -> Iterable[Optional[Mapping[str, Any]]]:
-        """
-        The `stream_slices` implements iterator functionality for `site_urls` and `searchType`. The user can pass many `site_url`,
-        and we have to process all of them, we can also pass the` searchType` parameter in the `request body` to get data using some`
-        searchType` value from [` web`, `news `,` image`, `video`,  `discover`, `googleNews`].
-        It's just a double nested loop with a yield statement.
-        """
-
-        for site_url in self._site_urls:
-            for search_type in self.search_types:
-                start_date = self._get_start_date(stream_state, site_url, search_type)
-                end_date = self._get_end_date()
-
-                if start_date > end_date:
-                    start_date = end_date
-
-                next_start = start_date
-                period = pendulum.Duration(days=self.range_of_days - 1)
-                while next_start <= end_date:
-                    next_end = min(next_start + period, end_date)
-                    yield {
-                        "site_url": site_url,
-                        "search_type": search_type,
-                        "start_date": next_start.to_date_string(),
-                        "end_date": next_end.to_date_string(),
-                        "data_state": self._data_state,
-                    }
-                    # add 1 day for the next slice's start date not to duplicate data from previous slice's end date.
-                    next_start = next_end + pendulum.Duration(days=1)
-
-    def next_page_token(self, response: requests.Response) -> Optional[bool]:
-        """
-        The `next_page_token` implements pagination functionality. This method gets the response
-        and compares the number of records with the constant `ROW_LIMITS` (maximum value 25000),
-        and if they are equal, this means that we get the end of the` Page`, and we need to go further,
-        for this we simply increase the `startRow` parameter in request body by `ROW_LIMIT` value.
-        """
-
-        if len(response.json().get(self.data_field, [])) == ROW_LIMIT:
-            self.start_row += ROW_LIMIT
-            return True
-
-        self.start_row = 0
-
-    def request_headers(self, **kwargs) -> Mapping[str, Any]:
-        return {"Content-Type": "application/json"}
-
-    def request_body_json(
-        self,
-        stream_state: Mapping[str, Any] = None,
-        stream_slice: Mapping[str, Any] = None,
-        next_page_token: Mapping[str, Any] = None,
-    ) -> Optional[Union[Dict[str, Any], str]]:
-        """
-        Here is a description of the parameters and implementations of the request body:
-        1. The `startDate` is retrieved from the `_get_start_date`,
-        if` SyncMode = full_refresh` just use `start_date` from configuration, otherwise use `get_update_state`.
-        2. The `endDate` is retrieved from the `config.json`.
-        3. The `sizes` parameter is used to group the result by some dimension.
-        The following dimensions are available: `date`, `country`, `page`, `device`, `query`.
-        4. For the `type` check the paragraph stream_slices method.
-         Filter results to the following type ["web", "news", "image", "video", "discover", "googleNews"]
-        5. For the `startRow` and `rowLimit` check next_page_token method.
-        """
-
-        data = {
-            "startDate": stream_slice["start_date"],
-            "endDate": stream_slice["end_date"],
-            "dimensions": self.dimensions,
-            "type": stream_slice.get("search_type"),
-            "aggregationType": self.aggregation_type.value,
-            "startRow": self.start_row,
-            "rowLimit": ROW_LIMIT,
-            "dataState": stream_slice.get("data_state"),
-        }
-
-        return data
-
-    def _get_end_date(self) -> pendulum.date:
-        end_date = pendulum.parse(self._end_date).date()
-        # limit `end_date` value with current date
-        return min(end_date, pendulum.now().date())
-
-    def _get_start_date(self, stream_state: Mapping[str, Any] = None, site_url: str = None, search_type: str = None) -> pendulum.date:
-        start_date = pendulum.parse(self._start_date)
-
-        if start_date and stream_state:
-            if stream_state.get(unquote_plus(site_url), {}).get(search_type):
-                stream_state_value = stream_state.get(unquote_plus(site_url), {}).get(search_type)
-
-                start_date = max(
-                    pendulum.parse(stream_state_value[self.cursor_field]),
-                    start_date,
-                )
-
-        return start_date.date()
-
-    def parse_response(
-        self,
-        response: requests.Response,
-        stream_state: Mapping[str, Any],
-        stream_slice: Mapping[str, Any] = None,
-        next_page_token: Mapping[str, Any] = None,
-    ) -> Iterable[Mapping]:
-        records = response.json().get(self.data_field) or []
-
-        for record in records:
-            record["site_url"] = unquote_plus(stream_slice.get("site_url"))
-            record["search_type"] = stream_slice.get("search_type")
-
-            for dimension in self.dimensions:
-                record[dimension] = record["keys"].pop(0)
-
-            # remove unnecessary empty field
-            record.pop("keys")
-
-            yield record
-
-    def _get_updated_state(
-        self,
-        current_stream_state: MutableMapping[str, Any],
-        latest_record: Mapping[str, Any],
-    ) -> Mapping[str, Any]:
-        """
-        With the existing nested loop implementation, we have to store a `cursor_field` for each `site_url`
-        and `searchType`. This functionality is placed in `get_update_state`.
-
-        {
-          "stream": {
-            "https://domain1.com": {
-              "web": {"date": "2022-01-03"},
-              "news": {"date": "2022-01-03"},
-              "image": {"date": "2022-01-03"},
-              "video": {"date": "2022-01-03"}
-            },
-            "https://domain2.com": {
-              "web": {"date": "2022-01-03"},
-              "news": {"date": "2022-01-03"},
-              "image": {"date": "2022-01-03"},
-              "video": {"date": "2022-01-03"}
-            },
-            "date": "2022-01-03",
-          }
-        }
-        """
-
-        latest_benchmark = latest_record.get(self.cursor_field)
-
-        site_url = latest_record.get("site_url")
-        search_type = latest_record.get("search_type")
-
-        value = current_stream_state.get(site_url, {}).get(search_type, {}).get(self.cursor_field)
-        if value:
-            latest_benchmark = max(latest_benchmark, value)
-        current_stream_state.setdefault(site_url, {}).setdefault(search_type, {})[self.cursor_field] = latest_benchmark
-
-        # we need to get the max date over all searchTypes but the current acceptance test YAML format doesn't
-        # support that
-        current_stream_state[self.cursor_field] = current_stream_state[site_url][search_type][self.cursor_field]
-
-        return current_stream_state
-
-    def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]:
-        for record in super().read_records(**kwargs):
-            self.state = self._get_updated_state(self.state, record)
-            yield record
-
-
-class SearchAnalyticsByCustomDimensions(SearchAnalytics):
-    # `date` is a cursor field therefore should be mandatory
-    DEFAULT_DIMENSIONS = ["date"]
-    DIMENSION_TO_PROPERTY_SCHEMA_MAP = {
-        "country": [{"country": {"type": ["null", "string"]}}],
-        "date": [{"date": {"type": ["null", "string"], "format": "date"}}],
-        "device": [{"device": {"type": ["null", "string"]}}],
-        "page": [{"page": {"type": ["null", "string"]}}],
-        "query": [{"query": {"type": ["null", "string"]}}],
-    }
-
-    primary_key = None
-
-    def __init__(self, dimensions: List[str], *args, **kwargs):
-        super(SearchAnalyticsByCustomDimensions, self).__init__(*args, **kwargs)
-        self.dimensions = dimensions + [dimension for dimension in self.DEFAULT_DIMENSIONS if dimension not in dimensions]
-        # Assign the dimensions as PK for the custom report stream.
-        # Site URL and Search Type are included in the API call thus affect the resulting data.
-        # `site_url` is a required URL param for making API calls;
-        # `search_type` remains a query param for historical reasons, we do not want to remove it to not break existing connections.
-        self.primary_key = self.dimensions + ["site_url", "search_type"]
-
-    def get_json_schema(self) -> Mapping[str, Any]:
-        schema: Mapping[str, Any] = {
-            "$schema": "https://json-schema.org/draft-07/schema#",
-            "type": ["null", "object"],
-            "additionalProperties": True,
-            "properties": {
-                # metrics
-                "clicks": {"type": ["null", "integer"]},
-                "ctr": {"type": ["null", "number"], "multipleOf": 1e-25},
-                "impressions": {"type": ["null", "integer"]},
-                "position": {"type": ["null", "number"], "multipleOf": 1e-25},
-                # default fields
-                "search_type": {"type": ["null", "string"]},
-                "site_url": {"type": ["null", "string"]},
-            },
-        }
-
-        # dimensions
-        dimension_properties = self.dimension_to_property_schema()
-        schema["properties"].update(dimension_properties)
-        return schema
-
-    def dimension_to_property_schema(self) -> dict:
-        properties = {}
-        for dimension in sorted(self.dimensions):
-            fields = self.DIMENSION_TO_PROPERTY_SCHEMA_MAP[dimension]
-            for field in fields:
-                properties = {**properties, **field}
-        return properties