airbyte-source-google-search-console 1.6.0.dev202503282247__py3-none-any.whl → 1.6.0rc1__py3-none-any.whl

{airbyte_source_google_search_console-1.6.0.dev202503282247.dist-info → airbyte_source_google_search_console-1.6.0rc1.dist-info}/METADATA

@@ -1,19 +1,19 @@
 Metadata-Version: 2.3
 Name: airbyte-source-google-search-console
-Version: 1.6.0.dev202503282247
+Version: 1.6.0rc1
 Summary: Source implementation for Google Search Console.
 License: Elv2
 Author: Airbyte
 Author-email: contact@airbyte.io
-Requires-Python: >=3.9,<3.12
+Requires-Python: >=3.10,<3.12
 Classifier: License :: Other/Proprietary License
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: airbyte-cdk (>=1,<2)
+Requires-Dist: airbyte-cdk (>=6,<7)
 Requires-Dist: google-api-python-client (==2.105.0)
 Requires-Dist: google-auth (==2.23.3)
+Requires-Dist: pendulum (>=3.1.0,<4.0.0)
 Project-URL: Documentation, https://docs.airbyte.com/integrations/sources/google-search-console
 Project-URL: Homepage, https://airbyte.com
 Project-URL: Repository, https://github.com/airbytehq/airbyte

{airbyte_source_google_search_console-1.6.0.dev202503282247.dist-info → airbyte_source_google_search_console-1.6.0rc1.dist-info}/RECORD

@@ -1,29 +1,25 @@
 source_google_search_console/__init__.py,sha256=HQCPu-CK7XmVDtP9rmTdB2XyraVCc6pv9pw38-O8y48,1191
-source_google_search_console/config_migrations.py,sha256=9C8QoELMAxt8NHiEY4fZMCYy5PwoOSUd6vSwoC1B3FM,4135
+source_google_search_console/components.py,sha256=5o8kH2xwYUvk3yjnSd6okJVF2KBUqUIF4V97xRpPAyI,2803
+source_google_search_console/config_migrations.py,sha256=Cl4SUdJpAf6wMM_vVhqjjU89NfUq9LIGJ9zNrWiBk-A,4235
 source_google_search_console/exceptions.py,sha256=iD3jYC4WxVCEKGsqQ7Vaj1tbjhJZ4S5mnSDnwFJdsIQ,1097
-source_google_search_console/run.py,sha256=q6O2iXoxtrdgtodCaanqTO2eKzUvXF2iDCfmCxaPE24,462
+source_google_search_console/manifest.yaml,sha256=zoPWyJ7gwOOK6yKAk4H92cD-N7Xg3iHtp5hTT_e80QA,10974
+source_google_search_console/run.py,sha256=TBkPlseTERarkj6wL8AMEKgm5Xsb2drnltPVH6257-M,2195
 source_google_search_console/schemas/search_analytics_all_fields.json,sha256=iQxRh_c_yz3uGofqpo1KX571TMmzYjKScb0PtI6SN_Q,1729
-source_google_search_console/schemas/search_analytics_by_country.json,sha256=xvUVjGRy63dsc7c0O-Kg7DUzybRpD-r_-VYhPmDBw_o,1491
 source_google_search_console/schemas/search_analytics_by_date.json,sha256=meCbWDayc1y0q-Lu-CAdjQVnsM8xZBX3BdF129UC1P8,1388
 source_google_search_console/schemas/search_analytics_by_device.json,sha256=VtoFjmmv9rx-uhSFaRn0wm4LeSxRIaexrxg2Spvbneo,1525
 source_google_search_console/schemas/search_analytics_by_page.json,sha256=KyUojZc4Lv3hPswxIJzUL5QDNsbvSugGjl_uHGF7Am4,1473
 source_google_search_console/schemas/search_analytics_by_query.json,sha256=mYc1Fu1A5TWLZCApZSjjfs_urW5BRUCPQI3rw3yQjt4,1439
 source_google_search_console/schemas/search_analytics_keyword_page_report.json,sha256=hCQZbpW9UNiCOrnUMkj4oQdzY0k8Hqoh1XkuKHvyvZw,1681
-source_google_search_console/schemas/search_analytics_keyword_page_report_minimal_dimensions.json,sha256=hCQZbpW9UNiCOrnUMkj4oQdzY0k8Hqoh1XkuKHvyvZw,1681
 source_google_search_console/schemas/search_analytics_keyword_site_report_by_page.json,sha256=lFpbShmTQhFyySc_JCdIOqMP9RfOIA15fpQN9xNXtTI,1626
-source_google_search_console/schemas/search_analytics_keyword_site_report_by_page_minimal_dimensions.json,sha256=lFpbShmTQhFyySc_JCdIOqMP9RfOIA15fpQN9xNXtTI,1626
 source_google_search_console/schemas/search_analytics_keyword_site_report_by_site.json,sha256=AVUl5x9jLL0FcStbTR-FMy-_A7uM-VlrJ-kdCBXWz_g,1755
-source_google_search_console/schemas/search_analytics_keyword_site_report_by_site_minimal_dimensions.json,sha256=AVUl5x9jLL0FcStbTR-FMy-_A7uM-VlrJ-kdCBXWz_g,1755
 source_google_search_console/schemas/search_analytics_page_report.json,sha256=-b0Y0LenTchS0q9A2aQ4hIjUjXkYF8erOtyrTMhf6MM,1776
 source_google_search_console/schemas/search_analytics_site_report_by_page.json,sha256=hWKHkm1reqGGu1dNcWBe6_XkZ5tK-UaiymrYRVgxRxI,1515
 source_google_search_console/schemas/search_analytics_site_report_by_site.json,sha256=rAh6LuNy7nCrrNM9MTd0qxAVc886ecQaqWRgV63OfyA,1408
-source_google_search_console/schemas/sitemaps.json,sha256=coyPSZCAfzMheybfRp4WPAZCp5JF2KGRF2rWK8oC080,1775
-source_google_search_console/schemas/sites.json,sha256=WNiCRuStPL1YkJiFa8FEbNJmqaERAOf9Yow6ygIumvo,383
-source_google_search_console/service_account_authenticator.py,sha256=gjUxt0xFxj82uviCQNTsA1Jlee__UDhYNjE7bRO1G0U,1227
-source_google_search_console/source.py,sha256=4UJW4KUVGo5WxH9ewWBVSemXH17Iqp-eaNRtUfLNET8,10431
+source_google_search_console/service_account_authenticator.py,sha256=pAWKAXfwfTY3xkXvQJH0EyFphFULdCIcC47YXYTO9X8,1307
+source_google_search_console/source.py,sha256=7FD4ciRrsptU7ZIxAU2xLC37bgjKWzNkflE9ybmgpXM,9113
 source_google_search_console/spec.json,sha256=WYtFvaSqWYGm1Dt2yV9G92U78Q94rh9oarbxJe3H7xo,8470
-source_google_search_console/streams.py,sha256=1rgtlcVUyAP3oKRNQ34qIh6rwTzvlr636GiEy0RPQLs,20508
-airbyte_source_google_search_console-1.6.0.dev202503282247.dist-info/METADATA,sha256=YyTR_gsHe3OcxaiSStl2wtO8Wa5Vsfm16QJ6miZG8JA,5645
-airbyte_source_google_search_console-1.6.0.dev202503282247.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
-airbyte_source_google_search_console-1.6.0.dev202503282247.dist-info/entry_points.txt,sha256=DMcgc9bCX-Vt6hm_68pa77qS3eGdeMhg-UdlFc-XKUM,85
-airbyte_source_google_search_console-1.6.0.dev202503282247.dist-info/RECORD,,
+source_google_search_console/streams.py,sha256=T0eqhmxGPDAfFNMXFfG_vM4aYHFaHAHHAoj6s5XnjnI,18760
+airbyte_source_google_search_console-1.6.0rc1.dist-info/METADATA,sha256=smSxRuSxL5aXxK-OcjVGdpR_jLJN_-dMEtsKyMu5Oww,5624
+airbyte_source_google_search_console-1.6.0rc1.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+airbyte_source_google_search_console-1.6.0rc1.dist-info/entry_points.txt,sha256=DMcgc9bCX-Vt6hm_68pa77qS3eGdeMhg-UdlFc-XKUM,85
+airbyte_source_google_search_console-1.6.0rc1.dist-info/RECORD,,

{airbyte_source_google_search_console-1.6.0.dev202503282247.dist-info → airbyte_source_google_search_console-1.6.0rc1.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 2.1.1
+Generator: poetry-core 2.1.3
 Root-Is-Purelib: true
 Tag: py3-none-any

source_google_search_console/components.py

@@ -0,0 +1,81 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+from dataclasses import dataclass
+from typing import Any, Mapping, Optional
+
+from airbyte_cdk.sources.declarative.migrations.state_migration import StateMigration
+
+
+@dataclass
+class NestedSubstreamStateMigration(StateMigration):
+    """
+    We require a custom state migration because SearchAnalytics streams contain two nested levels of
+    substreams. The existing LegacyToPerPartitionStateMigration only handles one level.
+
+    Legacy state format is as follows:
+    {
+        "date": "2025-05-28",
+        "https://www.example.com/": {
+            "web": {
+                "date": "2025-05-25"
+            },
+            "news": {
+                "date": "2023-05-22"
+            }
+        }
+    }
+
+    The resulting migrated per-partition state is:
+    {
+        "use_global_cursor": false,
+        "states": [
+        {
+            "partition": {
+                "search_type": "web",
+                "site_url": "https://www.example.com/"
+            },
+            "cursor": {
+                "date": "2025-05-25"
+            }
+        },
+        {
+            "partition": {
+                "search_type": "news",
+                "site_url": "https://www.example.com/"
+            },
+            "cursor": {
+                "date": "2023-05-22"
+            }
+        }],
+        "state": {
+            "date": "2025-05-25"
+        }
+    }
+    """
+
+    def should_migrate(self, stream_state: Mapping[str, Any]) -> bool:
+        return len(stream_state) > 0 and "states" not in stream_state
+
+    def migrate(self, stream_state: Mapping[str, Any]) -> Mapping[str, Any]:
+        global_state: Optional[Mapping[str, Any]] = None
+        per_partition_state = []
+        for site_url_key, search_type_state in stream_state.items():
+            if site_url_key == "date":
+                # The legacy state also contains a global cursor value under the `date` key which equates
+                # to global state.
+                #
+                # However, the Python implementation does not appear to be implemented
+                # correctly and simply saves the state of the last seen partition. Since I don't trust the
+                # legacy value and in the current implementation global state is applied to partitions
+                # without an existing value, I'm making a conscious choice to not migrate the global value.
+                continue
+            else:
+                site_url = site_url_key
+                for search_type_key, cursor in search_type_state.items():
+                    per_partition_state.append({"partition": {"site_url": site_url, "search_type": search_type_key}, "cursor": cursor})
+        return {
+            "use_global_cursor": False,
+            "states": per_partition_state,
+        }

source_google_search_console/config_migrations.py

@@ -6,8 +6,11 @@
 import logging
 from typing import Any, List, Mapping
 
+import orjson
+
 from airbyte_cdk.config_observation import create_connector_config_control_message
 from airbyte_cdk.entrypoint import AirbyteEntrypoint
+from airbyte_cdk.models import AirbyteMessageSerializer
 from airbyte_cdk.sources import Source
 from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository
 
@@ -77,7 +80,7 @@ class MigrateCustomReports:
         cls.message_repository.emit_message(create_connector_config_control_message(migrated_config))
         # emit the Airbyte Control Message from message queue to stdout
         for message in cls.message_repository._message_queue:
-            print(message.json(exclude_unset=True))
+            print(orjson.dumps(AirbyteMessageSerializer.dump(message)).decode())
 
     @classmethod
     def migrate(cls, args: List[str], source: Source) -> None:

source_google_search_console/manifest.yaml

@@ -0,0 +1,341 @@
+version: 6.44.0
+
+type: DeclarativeSource
+
+check:
+  type: CheckStream
+  stream_names:
+    - sites
+
+definitions:
+  oauth_authenticator:
+    type: OAuthAuthenticator
+    client_id: "{{ config.get('authorization', {}).get('client_id') }}"
+    client_secret: "{{ config.get('authorization', {}).get('client_secret') }}"
+    refresh_token: "{{ config.get('authorization', {}).get('refresh_token') }}"
+    token_refresh_endpoint: "https://oauth2.googleapis.com/token"
+
+  jwt_profile_assertion_oauth_authenticator:
+    type: OAuthAuthenticator
+    token_refresh_endpoint: https://oauth2.googleapis.com/token
+    refresh_request_headers:
+      Content-Type: application/x-www-form-urlencoded
+    use_profile_assertion: true
+    profile_assertion:
+      type: JwtAuthenticator
+      secret_key: "{{ json_loads(config.get('authorization', {}).get('service_account_info', {})).get('private_key') }}"
+      algorithm: "RS256"
+      token_duration: 3600
+      jwt_payload:
+        aud: "{{ json_loads(config.get('authorization', {}).get('service_account_info', {})).get('token_uri') }}"
+        iss: "{{ json_loads(config.get('authorization', {}).get('service_account_info', {})).get('client_email') }}"
+      additional_jwt_payload:
+        scope: "https://www.googleapis.com/auth/webmasters.readonly"
+
+  selective_authenticator:
+    type: SelectiveAuthenticator
+    authenticator_selection_path: ["authorization", "auth_type"]
+    authenticators:
+      Client: "#/definitions/oauth_authenticator"
+      Service: "#/definitions/jwt_profile_assertion_oauth_authenticator"
+
+  search_analytics_by_country_stream:
+    type: DeclarativeStream
+    name: search_analytics_by_country
+    primary_key:
+      - site_url
+      - date
+      - country
+      - search_type
+    retriever:
+      type: SimpleRetriever
+      requester:
+        type: HttpRequester
+        url_base: https://www.googleapis.com/webmasters/v3
+        path: "/sites/{{  sanitize_url(stream_partition.get('site_url')) }}/searchAnalytics/query"
+        http_method: POST
+        authenticator: "#/definitions/selective_authenticator"
+        request_headers:
+          Content-Type: "application/json"
+        request_body_json:
+          startDate: "{{ stream_interval.get('start_time') }}"
+          endDate: "{{ stream_interval.get('end_time') }}"
+          dimensions: ["date", "country"]
+          type: "{{ stream_partition.get('search_type') }}"
+          aggregationType: auto
+          dataState: "{{ config.get('data_state', 'final') }}"
+        # Currently relying on the default error handler behavior. Two pieces of functionality not covered are
+        # - Silently skipping over 403 permissions errors and relying on partial success reporting
+        # - Retrying 400 errors with aggregation_type=auto instead of failing outright
+      paginator:
+        type: DefaultPaginator
+        page_token_option:
+          type: RequestOption
+          field_name: startRow
+          inject_into: body_json
+        page_size_option:
+          type: RequestOption
+          field_name: rowLimit
+          inject_into: body_json
+        pagination_strategy:
+          type: OffsetIncrement
+          page_size: 25000
+          inject_on_first_request: true
+      record_selector:
+        type: RecordSelector
+        extractor:
+          type: DpathExtractor
+          field_path:
+            - rows
+      partition_router:
+        - type: ListPartitionRouter
+          values: "{{ config['site_urls'] }}"
+          cursor_field: site_url
+        - type: ListPartitionRouter
+          values:
+            - web
+            - news
+            - image
+            - video
+            - discover
+            - googleNews
+          cursor_field: search_type
+    incremental_sync:
+      type: DatetimeBasedCursor
+      cursor_field: date
+      cursor_datetime_formats:
+        - "%Y-%m-%d"
+      datetime_format: "%Y-%m-%d"
+      start_datetime:
+        type: MinMaxDatetime
+        datetime: "{{ config.get('start_date', '2021-01-01') }}"
+        datetime_format: "%Y-%m-%d"
+      end_datetime:
+        type: MinMaxDatetime
+        datetime: "{{ config.get('end_date', today_utc()) }}"
+        datetime_format: "%Y-%m-%d"
+      step: P3D
+      cursor_granularity: P1D
+    transformations:
+      - type: AddFields
+        fields:
+          - path:
+              - site_url
+            value: "{{ stream_partition['site_url'] }}"
+          - path:
+              - search_type
+            value: "{{ stream_partition['search_type'] }}"
+      # The values in the 'keys' array in the record correspond to the same order that the dimensions
+      # are requested in the API request. For example, if the request body was `dimensions: ["date", "country"]`,
+      # then the first value of `keys` is placed under the `date` field. These arrays are always be the same length
+      # After extracting the keys, the `keys` array is removed from the record.
+      - type: AddFields
+        fields:
+          - path:
+              - date
+            value: "{{ record['keys'][0] }}"
+          - path:
+              - country
+            value: "{{ record['keys'][1] }}"
+      - type: RemoveFields
+        field_pointers:
+          - - keys
+    schema_loader:
+      type: InlineSchemaLoader
+      schema:
+        $ref: "#/schemas/search_analytics_by_country"
+    state_migrations:
+      - type: CustomStateMigration
+        class_name: source_google_search_console.components.NestedSubstreamStateMigration
+
+  sites_stream:
+    type: DeclarativeStream
+    name: sites
+    retriever:
+      type: SimpleRetriever
+      requester:
+        type: HttpRequester
+        url_base: https://www.googleapis.com/webmasters/v3
+        path: "/sites/{{ sanitize_url(stream_partition.get('site_url')) }}"
+        http_method: GET
+        authenticator: "#/definitions/selective_authenticator"
+      record_selector:
+        type: RecordSelector
+        extractor:
+          type: DpathExtractor
+          field_path: []
+      partition_router:
+        - type: ListPartitionRouter
+          values: "{{ config['site_urls'] }}"
+          cursor_field: site_url
+    schema_loader:
+      type: InlineSchemaLoader
+      schema:
+        $ref: "#/schemas/sites"
+
+  sitemaps_stream:
+    type: DeclarativeStream
+    name: sitemaps
+    retriever:
+      type: SimpleRetriever
+      requester:
+        type: HttpRequester
+        url_base: https://www.googleapis.com/webmasters/v3
+        path: "/sites/{{ sanitize_url(stream_partition.get('site_url')) }}/sitemaps"
+        http_method: GET
+        authenticator: "#/definitions/selective_authenticator"
+      record_selector:
+        type: RecordSelector
+        extractor:
+          type: DpathExtractor
+          field_path:
+            - "sitemap"
+      partition_router:
+        - type: ListPartitionRouter
+          values: "{{ config['site_urls'] }}"
+          cursor_field: site_url
+    schema_loader:
+      type: InlineSchemaLoader
+      schema:
+        $ref: "#/schemas/sitemaps"
+
+streams:
+  - "#/definitions/search_analytics_by_country_stream"
+  - "#/definitions/sites_stream"
+  - "#/definitions/sitemaps_stream"
+
+schemas:
+  search_analytics_by_country:
+    $schema: "http://json-schema.org/draft-07/schema#"
+    type: object
+    properties:
+      site_url:
+        description: The URL of the site for which the search analytics data is being reported.
+        type:
+          - "null"
+          - string
+      search_type:
+        description: >-
+          The type of search (web search, image search, video search, etc.) for
+          which the data is being reported.
+        type:
+          - "null"
+          - string
+      date:
+        description: The date for which the search analytics data is being reported.
+        type:
+          - "null"
+          - string
+        format: date
+      country:
+        description: The country for which the search analytics data is being reported.
+        type:
+          - "null"
+          - string
+      clicks:
+        description: >-
+          The number of times users clicked on the search result for a specific
+          country.
+        type:
+          - "null"
+          - integer
+      impressions:
+        description: >-
+          The total number of times a search result was shown in search results for
+          a specific country.
+        type:
+          - "null"
+          - integer
+      ctr:
+        description: >-
+          The click-through rate, i.e., the ratio of clicks to impressions for a
+          specific country.
+        type:
+          - "null"
+          - number
+        multipleOf: 1.e-25
+      position:
+        description: >-
+          The average position at which the site's search result appeared for a
+          specific country.
+        type:
+          - "null"
+          - number
+        multipleOf: 1.e-25
+  sites:
+    $schema: "http://json-schema.org/draft-07/schema#"
+    type: object
+    properties:
+      siteUrl:
+        description: "The URL of the site data being fetched"
+        type: ["null", "string"]
+      permissionLevel:
+        description: "The user's permission level for the site (owner, full, restricted, etc.)"
+        type: ["null", "string"]
+  sitemaps:
+    $schema: "http://json-schema.org/draft-07/schema#"
+    type: object
+    properties:
+      path:
+        description: "Path to the sitemap file"
+        type:
+          - "null"
+          - string
+      lastSubmitted:
+        description: "Timestamp when the sitemap was last submitted"
+        type:
+          - "null"
+          - string
+        format: date-time
+      isPending:
+        description: "Flag indicating if the sitemap is pending for processing"
+        type:
+          - "null"
+          - boolean
+      isSitemapsIndex:
+        description: "Flag indicating if the data represents a sitemap index"
+        type:
+          - "null"
+          - boolean
+      type:
+        description: "Type of the sitemap"
+        type:
+          - "null"
+          - string
+      lastDownloaded:
+        description: "Timestamp when the sitemap was last downloaded"
+        type:
+          - "null"
+          - string
+        format: date-time
+      warnings:
+        description: "Warnings encountered while processing the sitemaps"
+        type:
+          - "null"
+          - string
+      errors:
+        description: "Errors encountered while processing the sitemaps"
+        type:
+          - "null"
+          - string
+      contents:
+        description: "Data related to the sitemap contents"
+        type: array
+        items:
+          type: object
+          properties:
+            type:
+              description: "Type of the sitemap content"
+              type:
+                - "null"
+                - string
+            submitted:
+              description: "Number of submitted sitemap URLs"
+              type:
+                - "null"
+                - string
+            indexed:
+              description: "Number of indexed sitemap URLs"
+              type:
+                - "null"
+                - string

source_google_search_console/run.py

@@ -1,18 +1,56 @@
 #
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
 #
 
-
 import sys
+import traceback
+from datetime import datetime
+from typing import List
+
+from orjson import orjson
 
-from airbyte_cdk.entrypoint import launch
+from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch, logger
+from airbyte_cdk.exception_handler import init_uncaught_exception_handler
+from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteMessageSerializer, AirbyteTraceMessage, TraceType, Type
 from source_google_search_console import SourceGoogleSearchConsole
 from source_google_search_console.config_migrations import MigrateCustomReports
 
 
-def run():
-    source = SourceGoogleSearchConsole()
-    # migrate config at runtime
-    MigrateCustomReports.migrate(sys.argv[1:], source)
-    # run the connector
-    launch(source, sys.argv[1:])
+def _get_source(args: List[str]):
+    catalog_path = AirbyteEntrypoint.extract_catalog(args)
+    config_path = AirbyteEntrypoint.extract_config(args)
+    state_path = AirbyteEntrypoint.extract_state(args)
+    try:
+        return SourceGoogleSearchConsole(
+            SourceGoogleSearchConsole.read_catalog(catalog_path) if catalog_path else None,
+            SourceGoogleSearchConsole.read_config(config_path) if config_path else None,
+            SourceGoogleSearchConsole.read_state(state_path) if state_path else None,
+        )
+    except Exception as error:
+        print(
+            orjson.dumps(
+                AirbyteMessageSerializer.dump(
+                    AirbyteMessage(
+                        type=Type.TRACE,
+                        trace=AirbyteTraceMessage(
+                            type=TraceType.ERROR,
+                            emitted_at=int(datetime.now().timestamp() * 1000),
+                            error=AirbyteErrorTraceMessage(
+                                message=f"Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance. Error: {error}",
+                                stack_trace=traceback.format_exc(),
+                            ),
+                        ),
+                    )
+                )
+            ).decode()
+        )
+        return None
+
+
+def run() -> None:
+    init_uncaught_exception_handler(logger)
+    _args = sys.argv[1:]
+    source = _get_source(_args)
+    if source:
+        MigrateCustomReports.migrate(sys.argv[1:], source)
+        launch(source, _args)

source_google_search_console/service_account_authenticator.py

@@ -1,6 +1,7 @@
 #
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
+import json
 
 import requests
 from google.auth.transport.requests import Request
@@ -21,7 +22,8 @@ class ServiceAccountAuthenticator(AuthBase):
 
     def __call__(self, request: requests.PreparedRequest) -> requests.PreparedRequest:
         try:
-            credentials: Credentials = Credentials.from_service_account_info(self.service_account_info, scopes=self.scopes).with_subject(
+            service_account_info = json.loads(self.service_account_info)
+            credentials: Credentials = Credentials.from_service_account_info(service_account_info, scopes=self.scopes).with_subject(
                 self.email
             )
             if not credentials.valid:

source_google_search_console/source.py

@@ -11,8 +11,9 @@ import jsonschema
 import pendulum
 import requests
 
-from airbyte_cdk.models import FailureType, SyncMode
-from airbyte_cdk.sources import AbstractSource
+from airbyte_cdk.models import ConfiguredAirbyteCatalog, FailureType, SyncMode
+from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource
+from airbyte_cdk.sources.source import TState
 from airbyte_cdk.sources.streams import Stream
 from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator
 from airbyte_cdk.utils import AirbyteTracedException
@@ -25,23 +26,17 @@ from source_google_search_console.exceptions import (
 from source_google_search_console.service_account_authenticator import ServiceAccountAuthenticator
 from source_google_search_console.streams import (
     SearchAnalyticsAllFields,
-    SearchAnalyticsByCountry,
     SearchAnalyticsByCustomDimensions,
     SearchAnalyticsByDate,
     SearchAnalyticsByDevice,
     SearchAnalyticsByPage,
     SearchAnalyticsByQuery,
     SearchAnalyticsKeywordPageReport,
-    SearchAnalyticsKeywordPageReportMinimalDimensions,
     SearchAnalyticsKeywordSiteReportByPage,
-    SearchAnalyticsKeywordSiteReportByPageMinimalDimensions,
     SearchAnalyticsKeywordSiteReportBySite,
-    SearchAnalyticsKeywordSiteReportBySiteMinimalDimensions,
     SearchAnalyticsPageReport,
     SearchAnalyticsSiteReportByPage,
     SearchAnalyticsSiteReportBySite,
-    Sitemaps,
-    Sites,
 )
 
 
@@ -58,7 +53,10 @@ custom_reports_schema = {
 }
 
 
-class SourceGoogleSearchConsole(AbstractSource):
+class SourceGoogleSearchConsole(YamlDeclarativeSource):
+    def __init__(self, catalog: Optional[ConfiguredAirbyteCatalog], config: Optional[Mapping[str, Any]], state: TState, **kwargs):
+        super().__init__(catalog=catalog, config=config, state=state, **{"path_to_yaml": "manifest.yaml"})
+
     @staticmethod
     def normalize_url(url):
         parse_result = urlparse(url)
@@ -71,7 +69,7 @@ class SourceGoogleSearchConsole(AbstractSource):
         authorization = config["authorization"]
         if authorization["auth_type"] == "Service":
             try:
-                authorization["service_account_info"] = json.loads(authorization["service_account_info"])
+                json.loads(authorization["service_account_info"])
             except ValueError:
                 message = "authorization.service_account_info is not valid JSON"
                 raise AirbyteTracedException(message=message, internal_message=message, failure_type=FailureType.config_error)
@@ -113,29 +111,6 @@ class SourceGoogleSearchConsole(AbstractSource):
                         raise AirbyteTracedException(message=message, internal_message=message, failure_type=FailureType.config_error)
         return config
 
-    def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Any]:
-        try:
-            config = self._validate_and_transform(config)
-            stream_kwargs = self.get_stream_kwargs(config)
-            self.validate_site_urls(config["site_urls"], stream_kwargs["authenticator"])
-            sites = Sites(**stream_kwargs)
-            stream_slice = sites.stream_slices(SyncMode.full_refresh)
-
-            # stream_slice returns all site_urls and we need to make sure that
-            # the connection is successful for all of them
-            for _slice in stream_slice:
-                sites_gen = sites.read_records(sync_mode=SyncMode.full_refresh, stream_slice=_slice)
-                next(sites_gen)
-            return True, None
-
-        except (InvalidSiteURLValidationError, UnauthorizedOauthError, UnauthorizedServiceAccountError, jsonschema.ValidationError) as e:
-            return False, repr(e)
-        except (Exception, UnidentifiedError) as error:
-            return (
-                False,
-                f"Unable to check connectivity to Google Search Console API - {repr(error)}",
-            )
-
     def validate_site_urls(self, site_urls: List[str], auth: Union[ServiceAccountAuthenticator, Oauth2Authenticator]):
         if isinstance(auth, ServiceAccountAuthenticator):
             request = auth(requests.Request(method="GET", url="https://www.googleapis.com/webmasters/v3/sites"))
@@ -154,7 +129,7 @@ class SourceGoogleSearchConsole(AbstractSource):
         if response.status_code != 200:
             raise UnidentifiedError(response.json())
 
-        remote_site_urls = {s["siteUrl"] for s in response.json()["siteEntry"]}
+        remote_site_urls = {s["siteUrl"] for s in response.json().get("siteEntry", [])}
         invalid_site_url = set(site_urls) - remote_site_urls
         if invalid_site_url:
             raise InvalidSiteURLValidationError(invalid_site_url)
@@ -175,25 +150,23 @@ class SourceGoogleSearchConsole(AbstractSource):
         config = self._validate_and_transform(config)
         stream_config = self.get_stream_kwargs(config)
 
-        streams = [
-            Sites(**stream_config),
-            Sitemaps(**stream_config),
-            SearchAnalyticsByCountry(**stream_config),
-            SearchAnalyticsByDevice(**stream_config),
-            SearchAnalyticsByDate(**stream_config),
-            SearchAnalyticsByQuery(**stream_config),
-            SearchAnalyticsByPage(**stream_config),
-            SearchAnalyticsAllFields(**stream_config),
-            SearchAnalyticsKeywordPageReport(**stream_config),
-            SearchAnalyticsKeywordPageReportMinimalDimensions(**stream_config),
-            SearchAnalyticsPageReport(**stream_config),
-            SearchAnalyticsSiteReportBySite(**stream_config),
-            SearchAnalyticsSiteReportByPage(**stream_config),
-            SearchAnalyticsKeywordSiteReportByPage(**stream_config),
-            SearchAnalyticsKeywordSiteReportByPageMinimalDimensions(**stream_config),
-            SearchAnalyticsKeywordSiteReportBySite(**stream_config),
-            SearchAnalyticsKeywordSiteReportBySiteMinimalDimensions(**stream_config),
-        ]
+        streams = super().streams(config=config)
+
+        streams.extend(
+            [
+                SearchAnalyticsByDevice(**stream_config),
+                SearchAnalyticsByDate(**stream_config),
+                SearchAnalyticsByQuery(**stream_config),
+                SearchAnalyticsByPage(**stream_config),
+                SearchAnalyticsAllFields(**stream_config),
+                SearchAnalyticsKeywordPageReport(**stream_config),
+                SearchAnalyticsPageReport(**stream_config),
+                SearchAnalyticsSiteReportBySite(**stream_config),
+                SearchAnalyticsSiteReportByPage(**stream_config),
+                SearchAnalyticsKeywordSiteReportByPage(**stream_config),
+                SearchAnalyticsKeywordSiteReportBySite(**stream_config),
+            ]
+        )
 
         streams = streams + self.get_custom_reports(config=config, stream_config=stream_config)
 

source_google_search_console/streams.py

@@ -81,40 +81,7 @@ class GoogleSearchConsole(HttpStream, ABC):
                 self.logger.error(f"Stream `{self.name}`. {error.get('message')}. Trying with `aggregationType = auto` instead.")
                 self.aggregation_type = QueryAggregationType.auto
                 setattr(self, "raise_on_http_errors", False)
-        return super().should_retry(response)
-
-
-class Sites(GoogleSearchConsole):
-    """
-    API docs: https://developers.google.com/webmaster-tools/search-console-api-original/v3/sites
-    """
-
-    primary_key = None
-
-    def path(
-        self,
-        stream_state: Mapping[str, Any] = None,
-        stream_slice: Mapping[str, Any] = None,
-        next_page_token: Mapping[str, Any] = None,
-    ) -> str:
-        return f"sites/{stream_slice.get('site_url')}"
-
-
-class Sitemaps(GoogleSearchConsole):
-    """
-    API docs: https://developers.google.com/webmaster-tools/search-console-api-original/v3/sitemaps
-    """
-
-    primary_key = None
-    data_field = "sitemap"
-
-    def path(
-        self,
-        stream_state: Mapping[str, Any] = None,
-        stream_slice: Mapping[str, Any] = None,
-        next_page_token: Mapping[str, Any] = None,
-    ) -> str:
-        return f"sites/{stream_slice.get('site_url')}/sitemaps"
+        return response.status_code == 429 or 500 <= response.status_code < 600
 
 
 class SearchAnalytics(GoogleSearchConsole, CheckpointMixin, ABC):
@@ -333,12 +300,6 @@ class SearchAnalyticsByDate(SearchAnalytics):
     dimensions = ["date"]
 
 
-class SearchAnalyticsByCountry(SearchAnalytics):
-    primary_key = ["site_url", "date", "country", "search_type"]
-    search_types = ["web", "news", "image", "video", "discover", "googleNews"]
-    dimensions = ["date", "country"]
-
-
 class SearchAnalyticsByDevice(SearchAnalytics):
     primary_key = ["site_url", "date", "device", "search_type"]
     search_types = ["web", "news", "image", "video", "googleNews"]
@@ -384,7 +345,7 @@ class SearchByKeyword(SearchAnalytics):
     def stream_slices(
         self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None
     ) -> Iterable[Optional[Mapping[str, Any]]]:
-        search_appearance_stream = SearchAppearance(self._session.auth, self._site_urls, self._start_date, self._end_date)
+        search_appearance_stream = SearchAppearance(self._http_client._session.auth, self._site_urls, self._start_date, self._end_date)
 
         for stream_slice in super().stream_slices(sync_mode, cursor_field, stream_state):
             keywords_records = search_appearance_stream.read_records(
@@ -395,6 +356,7 @@ class SearchByKeyword(SearchAnalytics):
             for keyword in keywords:
                 filters = {"dimension": "searchAppearance", "operator": "equals", "expression": keyword}
                 stream_slice["dimensionFilterGroups"] = [{"groupType": "and", "filters": filters}]
+
                 yield stream_slice
 
     def request_body_json(
@@ -413,35 +375,18 @@ class SearchAnalyticsKeywordPageReport(SearchByKeyword):
     dimensions = ["date", "country", "device", "query", "page"]
 
 
-class SearchAnalyticsKeywordPageReportMinimalDimensions(SearchByKeyword):
-    primary_key = ["site_url", "date", "country", "device", "query", "page", "search_type"]
-    dimensions = ["date"]
-
-
 class SearchAnalyticsKeywordSiteReportByPage(SearchByKeyword):
     primary_key = ["site_url", "date", "country", "device", "query", "search_type"]
     dimensions = ["date", "country", "device", "query"]
     aggregation_type = QueryAggregationType.by_page
 
 
-class SearchAnalyticsKeywordSiteReportByPageMinimalDimensions(SearchByKeyword):
-    primary_key = ["site_url", "date", "country", "device", "query", "search_type"]
-    dimensions = ["date"]
-    aggregation_type = QueryAggregationType.by_page
-
-
 class SearchAnalyticsKeywordSiteReportBySite(SearchByKeyword):
     primary_key = ["site_url", "date", "country", "device", "query", "search_type"]
     dimensions = ["date", "country", "device", "query"]
     aggregation_type = QueryAggregationType.by_property
 
 
-class SearchAnalyticsKeywordSiteReportBySiteMinimalDimensions(SearchByKeyword):
-    primary_key = ["site_url", "date", "country", "device", "query", "search_type"]
-    dimensions = ["date"]
-    aggregation_type = QueryAggregationType.by_property
-
-
 class SearchAnalyticsSiteReportBySite(SearchAnalytics):
     primary_key = ["site_url", "date", "country", "device", "search_type"]
     dimensions = ["date", "country", "device"]

source_google_search_console/schemas/search_analytics_by_country.json

@@ -1,41 +0,0 @@
-{
-  "$schema": "http://json-schema.org/draft-07/schema#",
-  "type": "object",
-  "properties": {
-    "site_url": {
-      "description": "The URL of the site for which the search analytics data is being reported.",
-      "type": ["null", "string"]
-    },
-    "search_type": {
-      "description": "The type of search (web search, image search, video search, etc.) for which the data is being reported.",
-      "type": ["null", "string"]
-    },
-    "date": {
-      "description": "The date for which the search analytics data is being reported.",
-      "type": ["null", "string"],
-      "format": "date"
-    },
-    "country": {
-      "description": "The country for which the search analytics data is being reported.",
-      "type": ["null", "string"]
-    },
-    "clicks": {
-      "description": "The number of times users clicked on the search result for a specific country.",
-      "type": ["null", "integer"]
-    },
-    "impressions": {
-      "description": "The total number of times a search result was shown in search results for a specific country.",
-      "type": ["null", "integer"]
-    },
-    "ctr": {
-      "description": "The click-through rate, i.e., the ratio of clicks to impressions for a specific country.",
-      "type": ["null", "number"],
-      "multipleOf": 1e-25
-    },
-    "position": {
-      "description": "The average position at which the site's search result appeared for a specific country.",
-      "type": ["null", "number"],
-      "multipleOf": 1e-25
-    }
-  }
-}

source_google_search_console/schemas/search_analytics_keyword_page_report_minimal_dimensions.json

@@ -1,54 +0,0 @@
-{
-  "$schema": "https://json-schema.org/draft-07/schema#",
-  "type": "object",
-  "additionalProperties": true,
-  "properties": {
-    "site_url": {
-      "description": "The URL of the website being monitored.",
-      "type": ["null", "string"]
-    },
-    "search_type": {
-      "description": "The type of search (e.g., web, image, video).",
-      "type": ["null", "string"]
-    },
-    "date": {
-      "description": "The date of the search data collected.",
-      "type": ["null", "string"],
-      "format": "date"
-    },
-    "country": {
-      "description": "The country where the search is made.",
-      "type": ["null", "string"]
-    },
-    "device": {
-      "description": "The device type used for the search (e.g., desktop, mobile).",
-      "type": ["null", "string"]
-    },
-    "page": {
-      "description": "The page URL on which the keyword appears in search results.",
-      "type": ["null", "string"]
-    },
-    "query": {
-      "description": "The search query used to find the site.",
-      "type": ["null", "string"]
-    },
-    "clicks": {
-      "description": "The number of clicks for the keyword on a specific page.",
-      "type": ["null", "integer"]
-    },
-    "impressions": {
-      "description": "The number of times the keyword appeared in search results.",
-      "type": ["null", "integer"]
-    },
-    "ctr": {
-      "description": "Click-through rate which is the percentage of clicks divided by impressions.",
-      "type": ["null", "number"],
-      "multipleOf": 1e-25
-    },
-    "position": {
-      "description": "The average position of the keyword on search results pages.",
-      "type": ["null", "number"],
-      "multipleOf": 1e-25
-    }
-  }
-}

source_google_search_console/schemas/search_analytics_keyword_site_report_by_page_minimal_dimensions.json

@@ -1,50 +0,0 @@
-{
-  "$schema": "https://json-schema.org/draft-07/schema#",
-  "type": "object",
-  "additionalProperties": true,
-  "properties": {
-    "site_url": {
-      "description": "The URL of the website for which the search analytics data is retrieved.",
-      "type": ["null", "string"]
-    },
-    "search_type": {
-      "description": "The type of search conducted (e.g., web, image, video).",
-      "type": ["null", "string"]
-    },
-    "date": {
-      "description": "The date when the search data was recorded.",
-      "type": ["null", "string"],
-      "format": "date"
-    },
-    "country": {
-      "description": "The country from which the search query originated.",
-      "type": ["null", "string"]
-    },
-    "device": {
-      "description": "The device type used for the search query (e.g., desktop, mobile).",
-      "type": ["null", "string"]
-    },
-    "query": {
-      "description": "The search query used by the user.",
-      "type": ["null", "string"]
-    },
-    "clicks": {
-      "description": "The number of times users clicked on your website link in search results.",
-      "type": ["null", "integer"]
-    },
-    "impressions": {
-      "description": "The number of times your website link appeared in search results.",
-      "type": ["null", "integer"]
-    },
-    "ctr": {
-      "description": "Click-through rate: Number of clicks divided by the number of impressions.",
-      "type": ["null", "number"],
-      "multipleOf": 1e-25
-    },
-    "position": {
-      "description": "The average position of your website link in search results.",
-      "type": ["null", "number"],
-      "multipleOf": 1e-25
-    }
-  }
-}

source_google_search_console/schemas/search_analytics_keyword_site_report_by_site_minimal_dimensions.json

@@ -1,50 +0,0 @@
-{
-  "$schema": "https://json-schema.org/draft-07/schema#",
-  "type": "object",
-  "additionalProperties": true,
-  "properties": {
-    "site_url": {
-      "description": "The URL of the site for which the search analytics data is recorded.",
-      "type": ["null", "string"]
-    },
-    "search_type": {
-      "description": "The type of search (e.g., web search, image search) that generated the analytics data.",
-      "type": ["null", "string"]
-    },
-    "date": {
-      "description": "The date for which the search analytics data is recorded.",
-      "type": ["null", "string"],
-      "format": "date"
-    },
-    "country": {
-      "description": "The country from which the search originated.",
-      "type": ["null", "string"]
-    },
-    "device": {
-      "description": "The type of device used by the user during the search (e.g., desktop, mobile).",
-      "type": ["null", "string"]
-    },
-    "query": {
-      "description": "The search query used by the user to find the site in search results.",
-      "type": ["null", "string"]
-    },
-    "clicks": {
-      "description": "The number of times users clicked on the search result linking to the site.",
-      "type": ["null", "integer"]
-    },
-    "impressions": {
-      "description": "The number of times the site was shown in search results to users.",
-      "type": ["null", "integer"]
-    },
-    "ctr": {
-      "description": "Click-through rate represents the percentage of users who clicked on the site's link after seeing it in search results.",
-      "type": ["null", "number"],
-      "multipleOf": 1e-25
-    },
-    "position": {
-      "description": "The average ranking position of the site in search results.",
-      "type": ["null", "number"],
-      "multipleOf": 1e-25
-    }
-  }
-}

source_google_search_console/schemas/sitemaps.json

@@ -1,61 +0,0 @@
-{
-  "$schema": "http://json-schema.org/draft-07/schema#",
-  "type": "object",
-  "properties": {
-    "path": {
-      "description": "Path to the sitemap file",
-      "type": ["null", "string"]
-    },
-    "lastSubmitted": {
-      "description": "Timestamp when the sitemap was last submitted",
-      "type": ["null", "string"],
-      "format": "date-time"
-    },
-    "isPending": {
-      "description": "Flag indicating if the sitemap is pending for processing",
-      "type": ["null", "boolean"]
-    },
-    "isSitemapsIndex": {
-      "description": "Flag indicating if the data represents a sitemap index",
-      "type": ["null", "boolean"]
-    },
-    "type": {
-      "description": "Type of the sitemap",
-      "type": ["null", "string"]
-    },
-    "lastDownloaded": {
-      "description": "Timestamp when the sitemap was last downloaded",
-      "type": ["null", "string"],
-      "format": "date-time"
-    },
-    "warnings": {
-      "description": "Warnings encountered while processing the sitemaps",
-      "type": ["null", "string"]
-    },
-    "errors": {
-      "description": "Errors encountered while processing the sitemaps",
-      "type": ["null", "string"]
-    },
-    "contents": {
-      "description": "Data related to the sitemap contents",
-      "type": "array",
-      "items": {
-        "type": "object",
-        "properties": {
-          "type": {
-            "description": "Type of the sitemap content",
-            "type": ["null", "string"]
-          },
-          "submitted": {
-            "description": "Number of submitted sitemap URLs",
-            "type": ["null", "string"]
-          },
-          "indexed": {
-            "description": "Number of indexed sitemap URLs",
-            "type": ["null", "string"]
-          }
-        }
-      }
-    }
-  }
-}

source_google_search_console/schemas/sites.json

@@ -1,14 +0,0 @@
-{
-  "$schema": "http://json-schema.org/draft-07/schema#",
-  "type": "object",
-  "properties": {
-    "siteUrl": {
-      "description": "The URL of the site data being fetched",
-      "type": ["null", "string"]
-    },
-    "permissionLevel": {
-      "description": "The user's permission level for the site (owner, full, restricted, etc.)",
-      "type": ["null", "string"]
-    }
-  }
-}