airbyte-source-google-search-console 1.6.0rc1__py3-none-any.whl → 1.8.0__py3-none-any.whl

source_google_search_console/manifest.yaml

@@ -39,14 +39,8 @@ definitions:
       Client: "#/definitions/oauth_authenticator"
       Service: "#/definitions/jwt_profile_assertion_oauth_authenticator"
 
-  search_analytics_by_country_stream:
+  base_search_analytics_stream:
     type: DeclarativeStream
-    name: search_analytics_by_country
-    primary_key:
-      - site_url
-      - date
-      - country
-      - search_type
     retriever:
       type: SimpleRetriever
       requester:
@@ -60,13 +54,10 @@ definitions:
         request_body_json:
           startDate: "{{ stream_interval.get('start_time') }}"
           endDate: "{{ stream_interval.get('end_time') }}"
-          dimensions: ["date", "country"]
+          dimensions: "{{ parameters['dimensions'] }}"
           type: "{{ stream_partition.get('search_type') }}"
-          aggregationType: auto
+          aggregationType: "{{ 'auto' if config.get('always_use_aggregation_type_auto') else parameters.get('aggregationType') }}"
           dataState: "{{ config.get('data_state', 'final') }}"
-        # Currently relying on the default error handler behavior. Two pieces of functionality not covered are
-        # - Silently skipping over 403 permissions errors and relying on partial success reporting
-        # - Retrying 400 errors with aggregation_type=auto instead of failing outright
       paginator:
         type: DefaultPaginator
         page_token_option:
@@ -92,13 +83,7 @@ definitions:
           values: "{{ config['site_urls'] }}"
           cursor_field: site_url
         - type: ListPartitionRouter
-          values:
-            - web
-            - news
-            - image
-            - video
-            - discover
-            - googleNews
+          values: "{{ parameters['search_types'] }}"
           cursor_field: search_type
     incremental_sync:
       type: DatetimeBasedCursor
@@ -116,6 +101,76 @@ definitions:
         datetime_format: "%Y-%m-%d"
       step: P3D
       cursor_granularity: P1D
+    state_migrations:
+      - type: CustomStateMigration
+        class_name: source_google_search_console.components.NestedSubstreamStateMigration
+
+  search_analytics_all_fields_stream:
+    $ref: "#/definitions/base_search_analytics_stream"
+    name: search_analytics_all_fields
+    primary_key:
+      - site_url
+      - date
+      - country
+      - device
+      - query
+      - page
+      - search_type
+    transformations:
+      - type: AddFields
+        fields:
+          - path:
+              - site_url
+            value: "{{ stream_partition['site_url'] }}"
+          - path:
+              - search_type
+            value: "{{ stream_partition['search_type'] }}"
+      - type: AddFields
+        fields:
+          - path:
+              - date
+            value: "{{ record['keys'][0] }}"
+          - path:
+              - country
+            value: "{{ record['keys'][1] }}"
+          - path:
+              - device
+            value: "{{ record['keys'][2] }}"
+          - path:
+              - page
+            value: "{{ record['keys'][3] }}"
+          - path:
+              - query
+            value: "{{ record['keys'][4] }}"
+      - type: RemoveFields
+        field_pointers:
+          - - keys
+    schema_loader:
+      type: InlineSchemaLoader
+      schema:
+        $ref: "#/schemas/search_analytics_all_fields"
+    $parameters:
+      dimensions:
+        - date
+        - country
+        - device
+        - page
+        - query
+      aggregationType: auto
+      search_types:
+        - web
+        - news
+        - image
+        - video
+
+  search_analytics_by_country_stream:
+    $ref: "#/definitions/base_search_analytics_stream"
+    name: search_analytics_by_country
+    primary_key:
+      - site_url
+      - date
+      - country
+      - search_type
     transformations:
       - type: AddFields
         fields:
@@ -144,9 +199,345 @@ definitions:
       type: InlineSchemaLoader
       schema:
         $ref: "#/schemas/search_analytics_by_country"
-    state_migrations:
-      - type: CustomStateMigration
-        class_name: source_google_search_console.components.NestedSubstreamStateMigration
+    $parameters:
+      dimensions:
+        - date
+        - country
+      aggregationType: auto
+      search_types:
+        - web
+        - news
+        - image
+        - video
+        - discover
+        - googleNews
+
+  search_analytics_by_date_stream:
+    $ref: "#/definitions/base_search_analytics_stream"
+    name: search_analytics_by_date
+    primary_key:
+      - site_url
+      - date
+      - search_type
+    transformations:
+      - type: AddFields
+        fields:
+          - path:
+              - site_url
+            value: "{{ stream_partition['site_url'] }}"
+          - path:
+              - search_type
+            value: "{{ stream_partition['search_type'] }}"
+      - type: AddFields
+        fields:
+          - path:
+              - date
+            value: "{{ record['keys'][0] }}"
+      - type: RemoveFields
+        field_pointers:
+          - - keys
+    schema_loader:
+      type: InlineSchemaLoader
+      schema:
+        $ref: "#/schemas/search_analytics_by_date"
+    $parameters:
+      dimensions:
+        - date
+      aggregationType: auto
+      search_types:
+        - web
+        - news
+        - image
+        - video
+        - discover
+        - googleNews
+
+  search_analytics_by_device_stream:
+    $ref: "#/definitions/base_search_analytics_stream"
+    name: search_analytics_by_device
+    primary_key:
+      - site_url
+      - date
+      - device
+      - search_type
+    transformations:
+      - type: AddFields
+        fields:
+          - path:
+              - site_url
+            value: "{{ stream_partition['site_url'] }}"
+          - path:
+              - search_type
+            value: "{{ stream_partition['search_type'] }}"
+      # The values in the 'keys' array in the record correspond to the same order that the dimensions
+      # are requested in the API request. For example, if the request body was `dimensions: ["date", "device"]`,
+      # then the first value of `keys` is placed under the `date` field. These arrays are always be the same length
+      # After extracting the keys, the `keys` array is removed from the record.
+      - type: AddFields
+        fields:
+          - path:
+              - date
+            value: "{{ record['keys'][0] }}"
+          - path:
+              - device
+            value: "{{ record['keys'][1] }}"
+      - type: RemoveFields
+        field_pointers:
+          - - keys
+    schema_loader:
+      type: InlineSchemaLoader
+      schema:
+        $ref: "#/schemas/search_analytics_by_device"
+    $parameters:
+      dimensions:
+        - date
+        - device
+      aggregationType: auto
+      search_types:
+        - web
+        - news
+        - image
+        - video
+        - googleNews
+
+  search_analytics_by_page_stream:
+    $ref: "#/definitions/base_search_analytics_stream"
+    name: search_analytics_by_page
+    primary_key:
+      - site_url
+      - date
+      - page
+      - search_type
+    transformations:
+      - type: AddFields
+        fields:
+          - path:
+              - site_url
+            value: "{{ stream_partition['site_url'] }}"
+          - path:
+              - search_type
+            value: "{{ stream_partition['search_type'] }}"
+      - type: AddFields
+        fields:
+          - path:
+              - date
+            value: "{{ record['keys'][0] }}"
+          - path:
+              - page
+            value: "{{ record['keys'][1] }}"
+      - type: RemoveFields
+        field_pointers:
+          - - keys
+    schema_loader:
+      type: InlineSchemaLoader
+      schema:
+        $ref: "#/schemas/search_analytics_by_page"
+    $parameters:
+      dimensions:
+        - date
+        - page
+      aggregationType: auto
+      search_types:
+        - web
+        - news
+        - image
+        - video
+        - discover
+        - googleNews
+
+  search_analytics_by_query_stream:
+    $ref: "#/definitions/base_search_analytics_stream"
+    name: search_analytics_by_query
+    primary_key:
+      - site_url
+      - date
+      - query
+      - search_type
+    transformations:
+      - type: AddFields
+        fields:
+          - path:
+              - site_url
+            value: "{{ stream_partition['site_url'] }}"
+          - path:
+              - search_type
+            value: "{{ stream_partition['search_type'] }}"
+      - type: AddFields
+        fields:
+          - path:
+              - date
+            value: "{{ record['keys'][0] }}"
+          - path:
+              - query
+            value: "{{ record['keys'][1] }}"
+      - type: RemoveFields
+        field_pointers:
+          - - keys
+    schema_loader:
+      type: InlineSchemaLoader
+      schema:
+        $ref: "#/schemas/search_analytics_by_query"
+    $parameters:
+      dimensions:
+        - date
+        - query
+      aggregationType: auto
+      search_types:
+        - web
+        - news
+        - image
+        - video
+
+  search_analytics_page_report_stream:
+    $ref: "#/definitions/base_search_analytics_stream"
+    name: search_analytics_page_report
+    primary_key:
+      - site_url
+      - date
+      - country
+      - device
+      - search_type
+      - page
+    transformations:
+      - type: AddFields
+        fields:
+          - path:
+              - site_url
+            value: "{{ stream_partition.get('site_url') }}"
+          - path:
+              - search_type
+            value: "{{ stream_partition.get('search_type') }}"
+      - type: AddFields
+        fields:
+          - path:
+              - date
+            value: "{{ record['keys'][0] }}"
+          - path:
+              - country
+            value: "{{ record['keys'][1] }}"
+          - path:
+              - device
+            value: "{{ record['keys'][2] }}"
+          - path:
+              - page
+            value: "{{ record['keys'][3] }}"
+      - type: RemoveFields
+        field_pointers:
+          - - keys
+    schema_loader:
+      type: InlineSchemaLoader
+      schema:
+        $ref: "#/schemas/search_analytics_page_report"
+    $parameters:
+      dimensions:
+        - date
+        - country
+        - device
+        - page
+      search_types:
+        - web
+        - news
+        - image
+        - video
+        - googleNews
+
+  search_analytics_site_report_by_page_stream:
+    $ref: "#/definitions/base_search_analytics_stream"
+    name: search_analytics_site_report_by_page
+    primary_key:
+      - site_url
+      - date
+      - country
+      - device
+      - search_type
+    transformations:
+      - type: AddFields
+        fields:
+          - path:
+              - site_url
+            value: "{{ stream_partition.get('site_url') }}"
+          - path:
+              - search_type
+            value: "{{ stream_partition.get('search_type') }}"
+      - type: AddFields
+        fields:
+          - path:
+              - date
+            value: "{{ record['keys'][0] }}"
+          - path:
+              - country
+            value: "{{ record['keys'][1] }}"
+          - path:
+              - device
+            value: "{{ record['keys'][2] }}"
+      - type: RemoveFields
+        field_pointers:
+          - - keys
+    schema_loader:
+      type: InlineSchemaLoader
+      schema:
+        $ref: "#/schemas/search_analytics_site_report_by_page"
+    $parameters:
+      aggregationType: byPage
+      dimensions:
+        - date
+        - country
+        - device
+      search_types:
+        - web
+        - news
+        - image
+        - video
+        - googleNews
+
+  search_analytics_site_report_by_site_stream:
+    $ref: "#/definitions/base_search_analytics_stream"
+    name: search_analytics_site_report_by_site
+    primary_key:
+      - site_url
+      - date
+      - country
+      - device
+      - search_type
+    transformations:
+      - type: AddFields
+        fields:
+          - path:
+              - site_url
+            value: "{{ stream_partition.get('site_url') }}"
+          - path:
+              - search_type
+            value: "{{ stream_partition.get('search_type') }}"
+      - type: AddFields
+        fields:
+          - path:
+              - date
+            value: "{{ record['keys'][0] }}"
+          - path:
+              - country
+            value: "{{ record['keys'][1] }}"
+          - path:
+              - device
+            value: "{{ record['keys'][2] }}"
+      - type: RemoveFields
+        field_pointers:
+          - - keys
+    schema_loader:
+      type: InlineSchemaLoader
+      schema:
+        $ref: "#/schemas/search_analytics_site_report_by_site"
+    $parameters:
+      aggregationType: byProperty
+      dimensions:
+        - date
+        - country
+        - device
+      search_types:
+        - web
+        - news
+        - image
+        - video
+        - googleNews
 
   sites_stream:
     type: DeclarativeStream
@@ -199,65 +590,708 @@ definitions:
       schema:
         $ref: "#/schemas/sitemaps"
 
-streams:
-  - "#/definitions/search_analytics_by_country_stream"
-  - "#/definitions/sites_stream"
+  # This stream is only used as a parent stream for search_by_keyword substreams
+  search_appearances_stream:
+    type: DeclarativeStream
+    name: search_appearances
+    retriever:
+      type: SimpleRetriever
+      requester:
+        type: HttpRequester
+        url_base: https://www.googleapis.com/webmasters/v3
+        path: "/sites/{{ sanitize_url(stream_partition.get('site_url')) }}/searchAnalytics/query"
+        http_method: POST
+        authenticator: "#/definitions/selective_authenticator"
+        request_headers:
+          Content-Type: "application/json"
+        request_body_json:
+          startDate: "{{ config.get('start_date') }}"
+          endDate: "{{ config.get('end_date') }}"
+          dimensions: ["searchAppearance"]
+          type: "{{ stream_partition.get('search_type') }}"
+          aggregationType: auto
+          dataState: "{{ config.get('data_state', 'final') }}"
+      paginator:
+        type: DefaultPaginator
+        page_token_option:
+          type: RequestOption
+          field_name: startRow
+          inject_into: body_json
+        page_size_option:
+          type: RequestOption
+          field_name: rowLimit
+          inject_into: body_json
+        pagination_strategy:
+          type: OffsetIncrement
+          page_size: 25000
+          inject_on_first_request: true
+      record_selector:
+        type: RecordSelector
+        extractor:
+          type: DpathExtractor
+          field_path:
+            - rows
+      partition_router:
+        - type: ListPartitionRouter
+          values: "{{ config['site_urls'] }}"
+          cursor_field: site_url
+        - type: ListPartitionRouter
+          values:
+            - web
+            - news
+            - image
+            - video
+          cursor_field: search_type
+    transformations:
+      - type: AddFields
+        fields:
+          - path:
+              - searchAppearance
+            value: "{{ record['keys'][0] }}"
+      - type: RemoveFields
+        field_pointers:
+          - - keys
+
+  base_search_analytics_keyword_stream:
+    type: DeclarativeStream
+    retriever:
+      type: SimpleRetriever
+      requester:
+        type: HttpRequester
+        url_base: https://www.googleapis.com/webmasters/v3
+        path: "/sites/{{  sanitize_url(stream_partition.get('site_url')) }}/searchAnalytics/query"
+        http_method: POST
+        authenticator: "#/definitions/selective_authenticator"
+        request_headers:
+          Content-Type: "application/json"
+        request_body_json:
+          startDate: "{{ stream_interval.get('start_time') }}"
+          endDate: "{{ stream_interval.get('end_time') }}"
+          dimensions: "{{ parameters.get('dimensions') }}"
+          type: "{{ stream_partition.get('parent_slice', {}).get('search_type') }}"
+          aggregationType: "{{ 'auto' if config.get('always_use_aggregation_type_auto') else parameters.get('aggregationType') }}"
+          dataState: "{{ config.get('data_state', 'final') }}"
+          dimensionFilterGroups: "{{ [{'groupType': 'and', 'filters': {'dimension': 'searchAppearance', 'operator': 'equals', 'expression': stream_partition.get('search_appearance')}}] }}"
+        error_handler:
+          type: DefaultErrorHandler
+          response_filters:
+            - type: HttpResponseFilter
+              action: FAIL
+              http_codes:
+                - 400
+              error_message: >-
+                Invalid aggregationType '{{ parameters.get('aggregationType') }}' used in the body of the API request. If you see this error, enable the
+                'always_use_aggregation_type_auto' config setting which will automatically use aggregationType=auto
+      paginator:
+        type: DefaultPaginator
+        page_token_option:
+          type: RequestOption
+          field_name: startRow
+          inject_into: body_json
+        page_size_option:
+          type: RequestOption
+          field_name: rowLimit
+          inject_into: body_json
+        pagination_strategy:
+          type: OffsetIncrement
+          page_size: 25000
+          inject_on_first_request: true
+      record_selector:
+        type: RecordSelector
+        extractor:
+          type: DpathExtractor
+          field_path:
+            - rows
+      partition_router:
+        - type: ListPartitionRouter
+          values: "{{ config['site_urls'] }}"
+          cursor_field: site_url
+        - type: SubstreamPartitionRouter
+          parent_stream_configs:
+            - type: ParentStreamConfig
+              parent_key: searchAppearance
+              partition_field: search_appearance
+              stream:
+                $ref: "#/definitions/search_appearances_stream"
+    incremental_sync:
+      type: DatetimeBasedCursor
+      cursor_field: date
+      cursor_datetime_formats:
+        - "%Y-%m-%d"
+      datetime_format: "%Y-%m-%d"
+      start_datetime:
+        type: MinMaxDatetime
+        datetime: "{{ config.get('start_date', '2021-01-01') }}"
+        datetime_format: "%Y-%m-%d"
+      end_datetime:
+        type: MinMaxDatetime
+        datetime: "{{ config.get('end_date', today_utc()) }}"
+        datetime_format: "%Y-%m-%d"
+      step: P3D
+      cursor_granularity: P1D
+
+  search_analytics_keyword_page_report_stream:
+    $ref: "#/definitions/base_search_analytics_keyword_stream"
+    name: search_analytics_keyword_page_report
+    primary_key:
+      - site_url
+      - date
+      - country
+      - device
+      - query
+      - page
+      - search_type
+    transformations:
+      - type: AddFields
+        fields:
+          - path:
+              - site_url
+            value: "{{ stream_partition['site_url'] }}"
+          - path:
+              - search_type
+            value: "{{ stream_partition.get('parent_slice', {}).get('search_type') }}"
+      - type: AddFields
+        fields:
+          - path:
+              - date
+            value: "{{ record['keys'][0] }}"
+          - path:
+              - country
+            value: "{{ record['keys'][1] }}"
+          - path:
+              - device
+            value: "{{ record['keys'][2] }}"
+          - path:
+              - query
+            value: "{{ record['keys'][3] }}"
+          - path:
+              - page
+            value: "{{ record['keys'][4] }}"
+      - type: RemoveFields
+        field_pointers:
+          - - keys
+    schema_loader:
+      type: InlineSchemaLoader
+      schema:
+        $ref: "#/schemas/search_analytics_keyword_page_report"
+    $parameters:
+      aggregationType: auto
+      dimensions:
+        - date
+        - country
+        - device
+        - query
+        - page
+
+  search_analytics_keyword_site_report_by_page_stream:
+    $ref: "#/definitions/base_search_analytics_keyword_stream"
+    name: search_analytics_keyword_site_report_by_page
+    primary_key:
+      - site_url
+      - date
+      - country
+      - device
+      - query
+      - search_type
+    transformations:
+      - type: AddFields
+        fields:
+          - path:
+              - site_url
+            value: "{{ stream_partition['site_url'] }}"
+          - path:
+              - search_type
+            value: "{{ stream_partition.get('parent_slice', {}).get('search_type') }}"
+      - type: AddFields
+        fields:
+          - path:
+              - date
+            value: "{{ record['keys'][0] }}"
+          - path:
+              - country
+            value: "{{ record['keys'][1] }}"
+          - path:
+              - device
+            value: "{{ record['keys'][2] }}"
+          - path:
+              - query
+            value: "{{ record['keys'][3] }}"
+      - type: RemoveFields
+        field_pointers:
+          - - keys
+    schema_loader:
+      type: InlineSchemaLoader
+      schema:
+        $ref: "#/schemas/search_analytics_keyword_site_report_by_page"
+    $parameters:
+      aggregationType: byPage
+      dimensions:
+        - date
+        - country
+        - device
+        - query
+
+  search_analytics_keyword_site_report_by_site_stream:
+    $ref: "#/definitions/base_search_analytics_keyword_stream"
+    name: search_analytics_keyword_site_report_by_site
+    primary_key:
+      - site_url
+      - date
+      - country
+      - device
+      - query
+      - search_type
+    transformations:
+      - type: AddFields
+        fields:
+          - path:
+              - site_url
+            value: "{{ stream_partition['site_url'] }}"
+          - path:
+              - search_type
+            value: "{{ stream_partition.get('parent_slice', {}).get('search_type') }}"
+      - type: AddFields
+        fields:
+          - path:
+              - date
+            value: "{{ record['keys'][0] }}"
+          - path:
+              - country
+            value: "{{ record['keys'][1] }}"
+          - path:
+              - device
+            value: "{{ record['keys'][2] }}"
+          - path:
+              - query
+            value: "{{ record['keys'][3] }}"
+      - type: RemoveFields
+        field_pointers:
+          - - keys
+    schema_loader:
+      type: InlineSchemaLoader
+      schema:
+        $ref: "#/schemas/search_analytics_keyword_site_report_by_site"
+    $parameters:
+      aggregationType: byProperty
+      dimensions:
+        - date
+        - country
+        - device
+
+streams:
+  # Regular streams
+  - "#/definitions/sites_stream"
   - "#/definitions/sitemaps_stream"
+  # Search Analytics streams
+  - "#/definitions/search_analytics_all_fields_stream"
+  - "#/definitions/search_analytics_by_country_stream"
+  - "#/definitions/search_analytics_by_date_stream"
+  - "#/definitions/search_analytics_by_device_stream"
+  - "#/definitions/search_analytics_by_page_stream"
+  - "#/definitions/search_analytics_by_query_stream"
+  - "#/definitions/search_analytics_page_report_stream"
+  - "#/definitions/search_analytics_site_report_by_page_stream"
+  - "#/definitions/search_analytics_site_report_by_site_stream"
+  # Search Analytics Keyword streams
+  - "#/definitions/search_analytics_keyword_page_report_stream"
+  - "#/definitions/search_analytics_keyword_site_report_by_page_stream"
+  - "#/definitions/search_analytics_keyword_site_report_by_site_stream"
+
+# Google Search Console has three layers of quotas that dictate rate limiting at the
+# user making requests, site being requested, and developer console key used.
+# https://developers.google.com/webmaster-tools/limits#qps-quota
+# - Per Site Quota: 1,200 req/min (20 req/sec)
+# - Per User Quota: 1,200 req/min (20 req/sec)
+# - Per Project Quota: 30,000,000 req/day (350 req/sec) / 40,000 req/min (60 req/sec)
+#
+# The most likely upper bound is based on the user quota since it is the lowest and the
+# same authenticated user account may hit multiple site urls. The default is set to 40
+# which equates to one request every 2 seconds which seems like a fair baseline.
+#
+concurrency_level:
+  type: ConcurrencyLevel
+  default_concurrency: "{{ config.get('num_workers', 40) }}"
+  max_concurrency: 100
 
 schemas:
+  search_analytics_all_fields:
+    $schema: "http://json-schema.org/draft-07/schema#"
+    type: object
+    properties:
+      site_url:
+        description: "The URL of the site from which the data originates."
+        type: ["null", "string"]
+      search_type:
+        description: "The type of search (e.g., web, image, video) that triggered the search result."
+        type: ["null", "string"]
+      date:
+        description: "The date when the search query occurred."
+        type: ["null", "string"]
+        format: "date"
+      country:
+        description: "The country from which the search query originated."
+        type: ["null", "string"]
+      device:
+        description: "The type of device used by the user (e.g., desktop, mobile)."
+        type: ["null", "string"]
+      page:
+        description: "The page URL that appeared in the search results."
+        type: ["null", "string"]
+      query:
+        description: "The search query entered by the user."
+        type: ["null", "string"]
+      clicks:
+        description: "The number of times users clicked on the search result for a specific query."
+        type: ["null", "integer"]
+      impressions:
+        description: "The number of times a search result appeared in response to a query."
+        type: ["null", "integer"]
+      ctr:
+        description: "Click-through rate, calculated as clicks divided by impressions."
+        type: ["null", "number"]
+        multipleOf: 1.e-25
+      position:
+        description: "The average position of the search result on the search engine results page."
+        type: ["null", "number"]
+        multipleOf: 1.e-25
   search_analytics_by_country:
     $schema: "http://json-schema.org/draft-07/schema#"
     type: object
     properties:
       site_url:
-        description: The URL of the site for which the search analytics data is being reported.
+        description: The URL of the site for which the search analytics data is being reported.
+        type:
+          - "null"
+          - string
+      search_type:
+        description: >-
+          The type of search (web search, image search, video search, etc.) for
+          which the data is being reported.
+        type:
+          - "null"
+          - string
+      date:
+        description: The date for which the search analytics data is being reported.
+        type:
+          - "null"
+          - string
+        format: date
+      country:
+        description: The country for which the search analytics data is being reported.
+        type:
+          - "null"
+          - string
+      clicks:
+        description: >-
+          The number of times users clicked on the search result for a specific
+          country.
+        type:
+          - "null"
+          - integer
+      impressions:
+        description: >-
+          The total number of times a search result was shown in search results for
+          a specific country.
+        type:
+          - "null"
+          - integer
+      ctr:
+        description: >-
+          The click-through rate, i.e., the ratio of clicks to impressions for a
+          specific country.
+        type:
+          - "null"
+          - number
+        multipleOf: 1.e-25
+      position:
+        description: >-
+          The average position at which the site's search result appeared for a
+          specific country.
+        type:
+          - "null"
+          - number
+        multipleOf: 1.e-25
+  search_analytics_by_date:
+    $schema: "http://json-schema.org/draft-07/schema#"
+    type: object
+    properties:
+      site_url:
+        description: "The URL of the site for which the search analytics data is being reported."
+        type: ["null", "string"]
+      search_type:
+        description: "The type of search query (e.g., web, image, video) that generated the search analytics data."
+        type: ["null", "string"]
+      date:
+        description: "The date for which the search analytics data is being reported."
+        type: ["null", "string"]
+        format: "date"
+      clicks:
+        description: "The total number of times users clicked on the search result for the site URL on the specific date."
+        type: ["null", "integer"]
+      impressions:
+        description: "The number of times the site URL was displayed in the search results to users on the specific date."
+        type: ["null", "integer"]
+      ctr:
+        description: "The click-through rate (CTR) represents the percentage of total impressions that resulted in a click to the site URL."
+        type: ["null", "number"]
+        multipleOf: 1.e-25
+      position:
+        description: "The average position of the site URL in the search results pages for the specific date."
+        type: ["null", "number"]
+        multipleOf: 1.e-25
+  search_analytics_by_device:
+    $schema: "http://json-schema.org/draft-07/schema#"
+    type: "object"
+    properties:
+      site_url:
+        description: "The URL of the site for which search analytics data is being provided."
+        type: ["null", "string"]
+      search_type:
+        description: "The type of search performed (e.g., web search, image search, video search)."
+        type: ["null", "string"]
+      date:
+        description: "The date for which the search analytics data is provided."
+        type: ["null", "string"]
+        format: "date"
+      device:
+        description: "The type of device used by the user for the search query (e.g., desktop, mobile)."
+        type: ["null", "string"]
+      clicks:
+        description: "The total number of times a user clicked on a search result linking to the target site."
+        type: ["null", "integer"]
+      impressions:
+        description: "The total number of times a user saw a link to the target site in search results."
+        type: ["null", "integer"]
+      ctr:
+        description: "Click-through rate represents the ratio of clicks to impressions, showing the effectiveness of your site in attracting clicks from search results."
+        type: ["null", "number"]
+        multipleOf: 1.e-25
+      position:
+        description: "The average position of the site's URLs in search results for the given query or queries."
+        type: ["null", "number"]
+        multipleOf: 1.e-25
+  search_analytics_by_page:
+    $schema: "http://json-schema.org/draft-07/schema#"
+    type: object
+    properties:
+      site_url:
+        description: "The URL of the site for which the search analytics data is being reported."
+        type: ["null", "string"]
+      search_type:
+        description: "The type of search query that led to the page being displayed in search results."
+        type: ["null", "string"]
+      date:
+        description: "The date for which the search analytics data is reported."
+        type: ["null", "string"]
+        format: "date"
+      page:
+        description: "The URL of the specific page being analyzed for search analytics data."
+        type: ["null", "string"]
+      clicks:
+        description: "The number of times a user clicked on the search result linking to the page."
+        type: ["null", "integer"]
+      impressions:
+        description: "The number of times a page from the site appeared in the search results viewed by users."
+        type: ["null", "integer"]
+      ctr:
+        description: "Click-through rate (CTR) is the ratio of clicks to impressions, indicating the effectiveness of the page in generating clicks."
+        type: ["null", "number"]
+        multipleOf: 1.e-25
+      position:
+        description: "The average position at which the page appeared in search results."
+        type: ["null", "number"]
+        multipleOf: 1.e-25
+  search_analytics_by_query:
+    $schema: "http://json-schema.org/draft-07/schema#"
+    type: object
+    properties:
+      site_url:
+        description: "The URL of the site for which the search analytics data is captured."
+        type: ["null", "string"]
+      search_type:
+        description: "The type of search result (e.g., web, image, video) for the specific query."
+        type: ["null", "string"]
+      date:
+        description: "The date for which the search analytics data is recorded."
+        type: ["null", "string"]
+        format: "date"
+      query:
+        description: "The search query for which the search analytics data is recorded."
+        type: ["null", "string"]
+      clicks:
+        description: "The number of times users clicked on the search result for the specific query."
+        type: ["null", "integer"]
+      impressions:
+        description: "The number of times the search result was displayed for the specific query."
+        type: ["null", "integer"]
+      ctr:
+        description: "The click-through rate (percentage) for the specific query, calculated as clicks divided by impressions."
+        type: ["null", "number"]
+        multipleOf: 1.e-25
+      position:
+        description: "The average position at which the search result appeared for the specific query."
+        type: ["null", "number"]
+        multipleOf: 1.e-25
+  search_analytics_page_report:
+    $schema: "https://json-schema.org/draft-07/schema#"
+    type: object
+    additionalProperties: true
+    properties:
+      site_url:
+        description: The URL of the website for which the search analytics data is being reported.
         type:
           - "null"
           - string
       search_type:
-        description: >-
-          The type of search (web search, image search, video search, etc.) for
-          which the data is being reported.
+        description: The type of search (e.g., web, image, video) that led users to the website.
         type:
           - "null"
           - string
       date:
-        description: The date for which the search analytics data is being reported.
+        description: The date when the search data was recorded.
         type:
           - "null"
           - string
         format: date
       country:
-        description: The country for which the search analytics data is being reported.
+        description: The country from which the search originated.
+        type:
+          - "null"
+          - string
+      page:
+        description: The specific page URL within the website that appeared in search results.
+        type:
+          - "null"
+          - string
+      device:
+        description: The type of device used by the user for the search query (e.g., desktop, mobile).
         type:
           - "null"
           - string
       clicks:
-        description: >-
-          The number of times users clicked on the search result for a specific
-          country.
+        description: The total number of times users clicked on search results that led to the linked website.
         type:
           - "null"
           - integer
       impressions:
-        description: >-
-          The total number of times a search result was shown in search results for
-          a specific country.
+        description: The total number of times a search result from the linked website was shown to users.
         type:
           - "null"
           - integer
       ctr:
-        description: >-
-          The click-through rate, i.e., the ratio of clicks to impressions for a
-          specific country.
+        description: "Click-through rate: The percentage of clicks out of the total impressions for a given search query."
         type:
           - "null"
           - number
         multipleOf: 1.e-25
       position:
-        description: >-
-          The average position at which the site's search result appeared for a
-          specific country.
+        description: The average position at which the website's search results appeared to users.
+        type:
+          - "null"
+          - number
+        multipleOf: 1.e-25
+  search_analytics_site_report_by_page:
+    $schema: "https://json-schema.org/draft-07/schema#"
+    type: object
+    additionalProperties: true
+    properties:
+      site_url:
+        description: The URL of the page on the site that is being reported.
+        type:
+          - "null"
+          - string
+      search_type:
+        description: The type of search query that led to the page being shown.
+        type:
+          - "null"
+          - string
+      date:
+        description: The date for which the data is being reported.
+        type:
+          - "null"
+          - string
+        format: date
+      country:
+        description: The country from which the search traffic originated.
+        type:
+          - "null"
+          - string
+      device:
+        description: "The type of device used by the searcher (e.g., desktop, mobile)."
+        type:
+          - "null"
+          - string
+      clicks:
+        description: The total number of clicks received by the page from search results.
+        type:
+          - "null"
+          - integer
+      impressions:
+        description: The total number of times the page appeared in search results.
+        type:
+          - "null"
+          - integer
+      ctr:
+        description: The click-through rate, i.e., the percentage of total impressions that resulted in clicks.
+        type:
+          - "null"
+          - number
+        multipleOf: 1.e-25
+      position:
+        description: The average position at which the page appeared in search results.
+        type:
+          - "null"
+          - number
+        multipleOf: 1.e-25
+  search_analytics_site_report_by_site:
+    $schema: "https://json-schema.org/draft-07/schema#"
+    type: object
+    additionalProperties: true
+    properties:
+      site_url:
+        description: The URL of the site being analyzed
+        type:
+          - "null"
+          - string
+      search_type:
+        description: "The type of search (e.g., web, image, video)"
+        type:
+          - "null"
+          - string
+      date:
+        description: The date of the search analytics data
+        type:
+          - "null"
+          - string
+        format: date
+      country:
+        description: The country where the search took place
+        type:
+          - "null"
+          - string
+      device:
+        description: "The type of device used for the search (e.g., mobile, desktop)"
+        type:
+          - "null"
+          - string
+      clicks:
+        description: The number of times users clicked on a search result linking to the site
+        type:
+          - "null"
+          - integer
+      impressions:
+        description: The number of times the site appeared in search results
+        type:
+          - "null"
+          - integer
+      ctr:
+        description: Click-through rate calculated as clicks divided by impressions
+        type:
+          - "null"
+          - number
+        multipleOf: 1.e-25
+      position:
+        description: The average position of the site in search results
         type:
           - "null"
           - number
@@ -339,3 +1373,195 @@ schemas:
               type:
                 - "null"
                 - string
+
+  search_analytics_keyword_page_report:
+    $schema: "https://json-schema.org/draft-07/schema#"
+    type: object
+    additionalProperties: true
+    properties:
+      site_url:
+        description: The URL of the website being monitored.
+        type:
+          - "null"
+          - string
+      search_type:
+        description: "The type of search (e.g., web, image, video)."
+        type:
+          - "null"
+          - string
+      date:
+        description: The date of the search data collected.
+        type:
+          - "null"
+          - string
+        format: date
+      country:
+        description: The country where the search is made.
+        type:
+          - "null"
+          - string
+      device:
+        description: "The device type used for the search (e.g., desktop, mobile)."
+        type:
+          - "null"
+          - string
+      page:
+        description: The page URL on which the keyword appears in search results.
+        type:
+          - "null"
+          - string
+      query:
+        description: The search query used to find the site.
+        type:
+          - "null"
+          - string
+      clicks:
+        description: The number of clicks for the keyword on a specific page.
+        type:
+          - "null"
+          - integer
+      impressions:
+        description: The number of times the keyword appeared in search results.
+        type:
+          - "null"
+          - integer
+      ctr:
+        description: >-
+          Click-through rate which is the percentage of clicks divided by
+          impressions.
+        type:
+          - "null"
+          - number
+        multipleOf: 1.e-25
+      position:
+        description: The average position of the keyword on search results pages.
+        type:
+          - "null"
+          - number
+        multipleOf: 1.e-25
+
+  search_analytics_keyword_site_report_by_page:
+    $schema: "https://json-schema.org/draft-07/schema#"
+    type: object
+    additionalProperties: true
+    properties:
+      site_url:
+        description: The URL of the website for which the search analytics data is retrieved.
+        type:
+          - "null"
+          - string
+      search_type:
+        description: "The type of search conducted (e.g., web, image, video)."
+        type:
+          - "null"
+          - string
+      date:
+        description: The date when the search data was recorded.
+        type:
+          - "null"
+          - string
+        format: date
+      country:
+        description: The country from which the search query originated.
+        type:
+          - "null"
+          - string
+      device:
+        description: "The device type used for the search query (e.g., desktop, mobile)."
+        type:
+          - "null"
+          - string
+      query:
+        description: The search query used by the user.
+        type:
+          - "null"
+          - string
+      clicks:
+        description: The number of times users clicked on your website link in search results.
+        type:
+          - "null"
+          - integer
+      impressions:
+        description: The number of times your website link appeared in search results.
+        type:
+          - "null"
+          - integer
+      ctr:
+        description: "Click-through rate: Number of clicks divided by the number of impressions."
+        type:
+          - "null"
+          - number
+        multipleOf: 1.e-25
+      position:
+        description: The average position of your website link in search results.
+        type:
+          - "null"
+          - number
+        multipleOf: 1.e-25
+
+  search_analytics_keyword_site_report_by_site:
+    $schema: "https://json-schema.org/draft-07/schema#"
+    type: object
+    additionalProperties: true
+    properties:
+      site_url:
+        description: The URL of the site for which the search analytics data is recorded.
+        type:
+          - "null"
+          - string
+      search_type:
+        description: >-
+          The type of search (e.g., web search, image search) that generated the
+          analytics data.
+        type:
+          - "null"
+          - string
+      date:
+        description: The date for which the search analytics data is recorded.
+        type:
+          - "null"
+          - string
+        format: date
+      country:
+        description: The country from which the search originated.
+        type:
+          - "null"
+          - string
+      device:
+        description: >-
+          The type of device used by the user during the search (e.g., desktop,
+          mobile).
+        type:
+          - "null"
+          - string
+      query:
+        description: The search query used by the user to find the site in search results.
+        type:
+          - "null"
+          - string
+      clicks:
+        description: >-
+          The number of times users clicked on the search result linking to the
+          site.
+        type:
+          - "null"
+          - integer
+      impressions:
+        description: The number of times the site was shown in search results to users.
+        type:
+          - "null"
+          - integer
+      ctr:
+        description: >-
+          Click-through rate represents the percentage of users who clicked on the
+          site's link after seeing it in search results.
+        type:
+          - "null"
+          - number
+        multipleOf: 1.e-25
+      position:
+        description: The average ranking position of the site in search results.
+        type:
+          - "null"
+          - number
+        multipleOf: 1.e-25