airbyte-source-google-sheets 0.8.4__py3-none-any.whl → 0.9.0rc1__py3-none-any.whl

source_google_sheets/source.py

@@ -1,277 +1,22 @@
 #
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
 #
 
+from typing import Any, Mapping, Optional
 
-import json
-import logging
-import socket
-from typing import Any, Generator, List, Mapping, MutableMapping, Optional, Union
+from airbyte_cdk.models import ConfiguredAirbyteCatalog
+from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource
+from airbyte_cdk.sources.source import TState
 
-from airbyte_cdk.models import FailureType
-from airbyte_cdk.models.airbyte_protocol import (
-    AirbyteCatalog,
-    AirbyteConnectionStatus,
-    AirbyteMessage,
-    AirbyteStateMessage,
-    AirbyteStreamStatus,
-    ConfiguredAirbyteCatalog,
-    Status,
-    Type,
-)
-from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
-from airbyte_cdk.sources.source import Source
-from airbyte_cdk.sources.streams.checkpoint import FullRefreshCheckpointReader
-from airbyte_cdk.utils import AirbyteTracedException
-from airbyte_cdk.utils.stream_status_utils import as_airbyte_message
-from apiclient import errors
-from google.auth import exceptions as google_exceptions
-from requests.status_codes import codes as status_codes
 
-from .client import GoogleSheetsClient
-from .helpers import Helpers
-from .models.spreadsheet import Spreadsheet
-from .models.spreadsheet_values import SpreadsheetValues
-from .utils import exception_description_by_status_code, safe_name_conversion
+"""
+This file provides the necessary constructs to interpret a provided declarative YAML configuration file into
+source connector.
+WARNING: Do not modify this file.
+"""
 
-# override default socket timeout to be 10 mins instead of 60 sec.
-# on behalf of https://github.com/airbytehq/oncall/issues/242
-DEFAULT_SOCKET_TIMEOUT: int = 600
-socket.setdefaulttimeout(DEFAULT_SOCKET_TIMEOUT)
 
-
-class SourceGoogleSheets(Source):
-    """
-    Spreadsheets API Reference: https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets
-    """
-
-    def check(self, logger: logging.Logger, config: json) -> AirbyteConnectionStatus:
-        # Check involves verifying that the specified spreadsheet is reachable with our credentials.
-        try:
-            client = GoogleSheetsClient(self.get_credentials(config))
-        except Exception as e:
-            return AirbyteConnectionStatus(status=Status.FAILED, message=f"Please use valid credentials json file. Error: {e}")
-
-        spreadsheet_id = Helpers.get_spreadsheet_id(config["spreadsheet_id"])
-
-        try:
-            spreadsheet = client.get(spreadsheetId=spreadsheet_id, includeGridData=False)
-        except errors.HttpError as err:
-            message = "Config error: "
-            # Give a clearer message if it's a common error like 404.
-            if err.resp.status == status_codes.NOT_FOUND:
-                message += "The spreadsheet link is not valid. Enter the URL of the Google spreadsheet you want to sync."
-            raise AirbyteTracedException(
-                message=message,
-                internal_message=message,
-                failure_type=FailureType.config_error,
-            ) from err
-        except google_exceptions.GoogleAuthError as err:
-            message = "Access to the spreadsheet expired or was revoked. Re-authenticate to restore access."
-            raise AirbyteTracedException(
-                message=message,
-                internal_message=message,
-                failure_type=FailureType.config_error,
-            ) from err
-
-        # Check for duplicate headers
-        spreadsheet_metadata = Spreadsheet.parse_obj(spreadsheet)
-        grid_sheets = Helpers.get_grid_sheets(spreadsheet_metadata)
-
-        duplicate_headers_in_sheet = {}
-        for sheet_name in grid_sheets:
-            try:
-                header_row_data = Helpers.get_first_row(client, spreadsheet_id, sheet_name)
-                if config.get("names_conversion"):
-                    header_row_data = [safe_name_conversion(h) for h in header_row_data]
-                _, duplicate_headers = Helpers.get_valid_headers_and_duplicates(header_row_data)
-                if duplicate_headers:
-                    duplicate_headers_in_sheet[sheet_name] = duplicate_headers
-            except Exception as err:
-                if str(err).startswith("Expected data for exactly one row for sheet"):
-                    logger.warn(f"Skip empty sheet: {sheet_name}")
-                else:
-                    logger.error(str(err))
-                    return AirbyteConnectionStatus(
-                        status=Status.FAILED, message=f"Unable to read the schema of sheet {sheet_name}. Error: {str(err)}"
-                    )
-        if duplicate_headers_in_sheet:
-            duplicate_headers_error_message = ", ".join(
-                [
-                    f"[sheet:{sheet_name}, headers:{duplicate_sheet_headers}]"
-                    for sheet_name, duplicate_sheet_headers in duplicate_headers_in_sheet.items()
-                ]
-            )
-            return AirbyteConnectionStatus(
-                status=Status.FAILED,
-                message="The following duplicate headers were found in the following sheets. Please fix them to continue: "
-                + duplicate_headers_error_message,
-            )
-
-        return AirbyteConnectionStatus(status=Status.SUCCEEDED)
-
-    def discover(self, logger: logging.Logger, config: json) -> AirbyteCatalog:
-        client = GoogleSheetsClient(self.get_credentials(config))
-        spreadsheet_id = Helpers.get_spreadsheet_id(config["spreadsheet_id"])
-        try:
-            logger.info(f"Running discovery on sheet {spreadsheet_id}")
-            spreadsheet_metadata = Spreadsheet.parse_obj(client.get(spreadsheetId=spreadsheet_id, includeGridData=False))
-            grid_sheets = Helpers.get_grid_sheets(spreadsheet_metadata)
-            streams = []
-            for sheet_name in grid_sheets:
-                try:
-                    header_row_data = Helpers.get_first_row(client, spreadsheet_id, sheet_name)
-                    if config.get("names_conversion"):
-                        header_row_data = [safe_name_conversion(h) for h in header_row_data]
-                    stream = Helpers.headers_to_airbyte_stream(logger, sheet_name, header_row_data)
-                    streams.append(stream)
-                except Exception as err:
-                    if str(err).startswith("Expected data for exactly one row for sheet"):
-                        logger.warn(f"Skip empty sheet: {sheet_name}")
-                    else:
-                        logger.error(str(err))
-            return AirbyteCatalog(streams=streams)
-
-        except errors.HttpError as err:
-            error_description = exception_description_by_status_code(err.resp.status, spreadsheet_id)
-            config_error_status_codes = [status_codes.NOT_FOUND, status_codes.FORBIDDEN]
-            if err.resp.status in config_error_status_codes:
-                message = f"{error_description}. {err.reason}."
-                raise AirbyteTracedException(
-                    message=message,
-                    internal_message=message,
-                    failure_type=FailureType.config_error,
-                ) from err
-            raise Exception(f"Could not discover the schema of your spreadsheet. {error_description}. {err.reason}.")
-        except google_exceptions.GoogleAuthError as err:
-            message = "Access to the spreadsheet expired or was revoked. Re-authenticate to restore access."
-            raise AirbyteTracedException(
-                message=message,
-                internal_message=message,
-                failure_type=FailureType.config_error,
-            ) from err
-
-    def _read(
-        self,
-        logger: logging.Logger,
-        config: json,
-        catalog: ConfiguredAirbyteCatalog,
-        state: Union[List[AirbyteStateMessage], MutableMapping[str, Any]] = None,
-    ) -> Generator[AirbyteMessage, None, None]:
-        client = GoogleSheetsClient(self.get_credentials(config))
-        client.Backoff.row_batch_size = config.get("batch_size", 200)
-
-        sheet_to_column_name = Helpers.parse_sheet_and_column_names_from_catalog(catalog)
-        stream_instances = {s.stream.name: s.stream for s in catalog.streams}
-        state_manager = ConnectorStateManager(stream_instance_map=stream_instances, state=state or {})
-        spreadsheet_id = Helpers.get_spreadsheet_id(config["spreadsheet_id"])
-
-        logger.info(f"Starting syncing spreadsheet {spreadsheet_id}")
-        # For each sheet in the spreadsheet, get a batch of rows, and as long as there hasn't been
-        # a blank row, emit the row batch
-        sheet_to_column_index_to_name = Helpers.get_available_sheets_to_column_index_to_name(
-            client, spreadsheet_id, sheet_to_column_name, config.get("names_conversion")
-        )
-        sheet_row_counts = Helpers.get_sheet_row_count(client, spreadsheet_id)
-        logger.info(f"Row counts: {sheet_row_counts}")
-        for sheet in sheet_to_column_index_to_name.keys():
-            logger.info(f"Syncing sheet {sheet}")
-            stream = stream_instances.get(sheet)
-            yield as_airbyte_message(stream, AirbyteStreamStatus.STARTED)
-            checkpoint_reader = FullRefreshCheckpointReader([])
-            _ = checkpoint_reader.next()
-            # We revalidate the sheet here to avoid errors in case the sheet was changed after the sync started
-            is_valid, reason = Helpers.check_sheet_is_valid(client, spreadsheet_id, sheet)
-            if not is_valid:
-                logger.info(f"Skipping syncing sheet {sheet}: {reason}")
-                yield self._checkpoint_state(checkpoint_reader.get_checkpoint(), state_manager, sheet, None)
-                yield as_airbyte_message(stream, AirbyteStreamStatus.INCOMPLETE)
-                continue
-
-            column_index_to_name = sheet_to_column_index_to_name[sheet]
-            row_cursor = 2  # we start syncing past the header row
-            # For the loop, it is necessary that the initial row exists when we send a request to the API,
-            # if the last row of the interval goes outside the sheet - this is normal, we will return
-            # only the real data of the sheet and in the next iteration we will loop out.
-            while row_cursor <= sheet_row_counts[sheet]:
-                row_batch = SpreadsheetValues.parse_obj(
-                    client.get_values(
-                        sheet=sheet,
-                        row_cursor=row_cursor,
-                        spreadsheetId=spreadsheet_id,
-                        majorDimension="ROWS",
-                    )
-                )
-
-                row_cursor += client.Backoff.row_batch_size + 1
-                # there should always be one range since we requested only one
-                value_ranges = row_batch.valueRanges[0]
-
-                if not value_ranges.values:
-                    break
-
-                row_values = value_ranges.values
-                if len(row_values) == 0:
-                    break
-
-                yield as_airbyte_message(stream, AirbyteStreamStatus.RUNNING)
-                for row in row_values:
-                    if not Helpers.is_row_empty(row) and Helpers.row_contains_relevant_data(row, column_index_to_name.keys()):
-                        yield AirbyteMessage(type=Type.RECORD, record=Helpers.row_data_to_record_message(sheet, row, column_index_to_name))
-
-            yield self._checkpoint_state(checkpoint_reader.get_checkpoint(), state_manager, sheet, None)
-            yield as_airbyte_message(stream, AirbyteStreamStatus.COMPLETE)
-
-    def _checkpoint_state(
-        self,
-        stream_state: Mapping[str, Any],
-        state_manager,
-        stream_name: str,
-        stream_namespace: Optional[str],
-    ) -> AirbyteMessage:
-        state_manager.update_state_for_stream(stream_name, stream_namespace, stream_state)
-        return state_manager.create_state_message(stream_name, stream_namespace)
-
-    def read(
-        self,
-        logger: logging.Logger,
-        config: json,
-        catalog: ConfiguredAirbyteCatalog,
-        state: Union[List[AirbyteStateMessage], MutableMapping[str, Any]] = None,
-    ) -> Generator[AirbyteMessage, None, None]:
-        spreadsheet_id = Helpers.get_spreadsheet_id(config["spreadsheet_id"])
-        try:
-            yield from self._read(logger, config, catalog, state)
-        except errors.HttpError as e:
-            error_description = exception_description_by_status_code(e.status_code, spreadsheet_id)
-
-            if e.status_code == status_codes.FORBIDDEN:
-                raise AirbyteTracedException(
-                    message=f"Stopped syncing process. {error_description}",
-                    internal_message=error_description,
-                    failure_type=FailureType.config_error,
-                ) from e
-            if e.status_code == status_codes.TOO_MANY_REQUESTS:
-                raise AirbyteTracedException(
-                    message=f"Stopped syncing process due to rate limits. {error_description}",
-                    internal_message=error_description,
-                    failure_type=FailureType.transient_error,
-                ) from e
-            else:
-                logger.info(f"{e.status_code}: {e.reason}. {error_description}")
-                raise AirbyteTracedException(
-                    message=f"Stopped syncing process. {error_description}",
-                    internal_message=error_description,
-                    failure_type=FailureType.transient_error,
-                ) from e
-        finally:
-            logger.info(f"Finished syncing spreadsheet {spreadsheet_id}")
-
-    @staticmethod
-    def get_credentials(config):
-        # backward compatible with old style config
-        if config.get("credentials_json"):
-            credentials = {"auth_type": "Service", "service_account_info": config.get("credentials_json")}
-            return credentials
-
-        return config.get("credentials")
+# Declarative Source
+class SourceGoogleSheets(YamlDeclarativeSource):
+    def __init__(self, catalog: Optional[ConfiguredAirbyteCatalog], config: Optional[Mapping[str, Any]], state: TState, **kwargs):
+        super().__init__(catalog=catalog, config=config, state=state, **{"path_to_yaml": "manifest.yaml"})

source_google_sheets/spec.yaml

@@ -12,17 +12,15 @@ connectionSpecification:
       type: integer
       title: Row Batch Size
       description: >-
-        Default value is 200.
+        Default value is 1000000.
         An integer representing row batch size for each sent request to Google Sheets API.
-        Row batch size means how many rows are processed from the google sheet, for example default value 200
-        would process rows 1-201, then 201-401 and so on.
+        Row batch size means how many rows are processed from the google sheet, for example default value 1000000
+        would process rows 2-1000002, then 1000003-2000003 and so on.
         Based on <a href='https://developers.google.com/sheets/api/limits'>Google Sheets API limits documentation</a>,
         it is possible to send up to 300 requests per minute, but each individual request has to be processed under 180 seconds,
         otherwise the request returns a timeout error. In regards to this information, consider network speed and
         number of columns of the google sheet when deciding a batch_size value.
-        Default value should cover most of the cases, but if a google sheet has over 100,000 records or more,
-        consider increasing batch_size value.
-      default: 200
+      default: 1000000
     spreadsheet_id:
       type: string
       title: Spreadsheet Link

source_google_sheets/utils.py

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
 #
 
 
@@ -8,6 +8,7 @@ import re
 import unidecode
 from requests.status_codes import codes as status_codes
 
+
 TOKEN_PATTERN = re.compile(r"[A-Z]+[a-z]*|[a-z]+|\d+|(?P<NoToken>[^a-zA-Z\d]+)")
 DEFAULT_SEPARATOR = "_"
 

airbyte_source_google_sheets-0.8.4.dist-info/RECORD

@@ -1,14 +0,0 @@
-source_google_sheets/__init__.py,sha256=-aGVMRfrgWjYad3_cHofIptEEa5WMQzTvFD92HevQfw,73
-source_google_sheets/client.py,sha256=dLujGTU2CPVV8WJwGHEBg3zJZK6xtzuyGTJm7TjAv9I,1916
-source_google_sheets/helpers.py,sha256=kKXop3YyQ3jPYlWgWW3GmRPxDmnuoHZ4joa3rQLCxUQ,10642
-source_google_sheets/models/__init__.py,sha256=Z-4MTpxG5t2jGhXzs4PPoIOa83zw3jRnUDx0N9Puv3s,61
-source_google_sheets/models/spreadsheet.py,sha256=fsHREpPEN36wCzGdqgfJ2EVW40UDZ_lS863A4XT2pGo,1112
-source_google_sheets/models/spreadsheet_values.py,sha256=y8ytuTqwpziJ2ICl0xhlRWgjMkxTfxOalRd414PMHZM,440
-source_google_sheets/run.py,sha256=_f5-LNqMzBuHtCD1YoUBxnA0fszgqmdNGcN7y_AmXU0,237
-source_google_sheets/source.py,sha256=KIcjUH_-vRNTwHd6rNv8c_jgPBJjXMMmcDvvVyng_OA,13504
-source_google_sheets/spec.yaml,sha256=WrPdH2xLCdyM-kY-pRqbwICcNPhv8nqnb2gdbslTsaQ,5141
-source_google_sheets/utils.py,sha256=ZB5lboyffiuuQdSarqe8AqBGEyiQpxiOfxqcU7Ght8A,2289
-airbyte_source_google_sheets-0.8.4.dist-info/METADATA,sha256=mIdbrQljTBznf31l7Xrr5Zy63tsNCI3LctSDj51bRhE,5539
-airbyte_source_google_sheets-0.8.4.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-airbyte_source_google_sheets-0.8.4.dist-info/entry_points.txt,sha256=Dtsfjohe5IPUFyqojk49SIoP7CifCTlNLG_pgivzppo,69
-airbyte_source_google_sheets-0.8.4.dist-info/RECORD,,

source_google_sheets/client.py

@@ -1,48 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-import logging
-from typing import Dict, List
-
-import backoff
-from googleapiclient import errors
-from requests import codes as status_codes
-
-from .helpers import SCOPES, Helpers
-
-logger = logging.getLogger("airbyte")
-
-
-class GoogleSheetsClient:
-    class Backoff:
-        row_batch_size = 200
-
-        @classmethod
-        def increase_row_batch_size(cls, details):
-            if details["exception"].status_code == status_codes.TOO_MANY_REQUESTS and cls.row_batch_size < 1000:
-                cls.row_batch_size = cls.row_batch_size + 100
-                logger.info(f"Increasing number of records fetching due to rate limits. Current value: {cls.row_batch_size}")
-
-        @staticmethod
-        def give_up(error):
-            code = error.resp.status
-            # Stop retrying if it's not a problem with the rate limit or on the server end
-            return not (code == status_codes.TOO_MANY_REQUESTS or 500 <= code < 600)
-
-    def __init__(self, credentials: Dict[str, str], scopes: List[str] = SCOPES):
-        self.client = Helpers.get_authenticated_sheets_client(credentials, scopes)
-
-    @backoff.on_exception(backoff.expo, errors.HttpError, max_time=120, giveup=Backoff.give_up, on_backoff=Backoff.increase_row_batch_size)
-    def get(self, **kwargs):
-        return self.client.get(**kwargs).execute()
-
-    @backoff.on_exception(backoff.expo, errors.HttpError, max_time=120, giveup=Backoff.give_up, on_backoff=Backoff.increase_row_batch_size)
-    def get_values(self, **kwargs):
-        range = self._create_range(kwargs.pop("sheet"), kwargs.pop("row_cursor"))
-        logger.info(f"Fetching range {range}")
-        return self.client.values().batchGet(ranges=range, **kwargs).execute()
-
-    def _create_range(self, sheet, row_cursor):
-        range = f"{sheet}!{row_cursor}:{row_cursor + self.Backoff.row_batch_size}"
-        return range

source_google_sheets/helpers.py

@@ -1,232 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-import json
-import logging
-import re
-from collections import defaultdict
-from datetime import datetime
-from typing import Dict, FrozenSet, Iterable, List, Tuple
-
-from airbyte_cdk.models.airbyte_protocol import AirbyteRecordMessage, AirbyteStream, ConfiguredAirbyteCatalog, SyncMode
-from google.oauth2 import credentials as client_account
-from google.oauth2 import service_account
-from googleapiclient import discovery
-
-from .models.spreadsheet import RowData, Spreadsheet
-from .utils import safe_name_conversion
-
-SCOPES = ["https://www.googleapis.com/auth/spreadsheets.readonly", "https://www.googleapis.com/auth/drive.readonly"]
-
-logger = logging.getLogger("airbyte")
-
-
-class Helpers(object):
-    @staticmethod
-    def get_authenticated_sheets_client(credentials: Dict[str, str], scopes: List[str] = SCOPES) -> discovery.Resource:
-        creds = Helpers.get_authenticated_google_credentials(credentials, scopes)
-        return discovery.build("sheets", "v4", credentials=creds).spreadsheets()
-
-    @staticmethod
-    def get_authenticated_drive_client(credentials: Dict[str, str], scopes: List[str] = SCOPES) -> discovery.Resource:
-        creds = Helpers.get_authenticated_google_credentials(credentials, scopes)
-        return discovery.build("drive", "v3", credentials=creds)
-
-    @staticmethod
-    def get_authenticated_google_credentials(credentials: Dict[str, str], scopes: List[str] = SCOPES):
-        auth_type = credentials.pop("auth_type")
-        if auth_type == "Service":
-            return service_account.Credentials.from_service_account_info(json.loads(credentials["service_account_info"]), scopes=scopes)
-        elif auth_type == "Client":
-            return client_account.Credentials.from_authorized_user_info(info=credentials)
-
-    @staticmethod
-    def headers_to_airbyte_stream(logger: logging.Logger, sheet_name: str, header_row_values: List[str]) -> AirbyteStream:
-        """
-        Parses sheet headers from the provided row. This method assumes that data is contiguous
-        i.e: every cell contains a value and the first cell which does not contain a value denotes the end
-        of the headers. For example, if the first row contains "One | Two | | Three" then this method
-        will parse the headers as ["One", "Two"]. This assumption is made for simplicity and can be modified later.
-        """
-        fields, duplicate_fields = Helpers.get_valid_headers_and_duplicates(header_row_values)
-        if duplicate_fields:
-            logger.warn(f"Duplicate headers found in {sheet_name}. Ignoring them: {duplicate_fields}")
-
-        sheet_json_schema = {
-            "$schema": "http://json-schema.org/draft-07/schema#",
-            "type": "object",
-            # For simplicity, the type of every cell is a string
-            "properties": {field: {"type": "string"} for field in fields},
-        }
-
-        return AirbyteStream(name=sheet_name, json_schema=sheet_json_schema, supported_sync_modes=[SyncMode.full_refresh])
-
-    @staticmethod
-    def get_valid_headers_and_duplicates(header_row_values: List[str]) -> (List[str], List[str]):
-        fields = []
-        duplicate_fields = set()
-        for cell_value in header_row_values:
-            if cell_value:
-                if cell_value in fields:
-                    duplicate_fields.add(cell_value)
-                else:
-                    fields.append(cell_value)
-            else:
-                break
-
-        # Removing all duplicate fields
-        if duplicate_fields:
-            fields = [field for field in fields if field not in duplicate_fields]
-
-        return fields, list(duplicate_fields)
-
-    @staticmethod
-    def get_formatted_row_values(row_data: RowData) -> List[str]:
-        """
-        Gets the formatted values of all cell data in this row. A formatted value is the final value a user sees in a spreadsheet.
-        It can be a raw string input by the user, or the result of a sheets function call.
-        """
-        return [value.formattedValue for value in row_data.values]
-
-    @staticmethod
-    def get_first_row(client, spreadsheet_id: str, sheet_name: str) -> List[str]:
-        spreadsheet = Spreadsheet.parse_obj(client.get(spreadsheetId=spreadsheet_id, includeGridData=True, ranges=f"{sheet_name}!1:1"))
-
-        # There is only one sheet since we are specifying the sheet in the requested ranges.
-        returned_sheets = spreadsheet.sheets
-        if len(returned_sheets) != 1:
-            raise Exception(f"Unexpected return result: Sheet {sheet_name} was expected to contain data on exactly 1 sheet. ")
-
-        range_data = returned_sheets[0].data
-        if len(range_data) != 1:
-            raise Exception(f"Expected data for exactly one range for sheet {sheet_name}")
-
-        all_row_data = range_data[0].rowData
-        if not all_row_data:
-            # the sheet is empty
-            logger.warning(f"The sheet {sheet_name} (ID {spreadsheet_id}) is empty!")
-            return []
-
-        if len(all_row_data) != 1:
-            raise Exception(f"Expected data for exactly one row for sheet {sheet_name}")
-
-        first_row_data = all_row_data[0]
-
-        return Helpers.get_formatted_row_values(first_row_data)
-
-    @staticmethod
-    def parse_sheet_and_column_names_from_catalog(catalog: ConfiguredAirbyteCatalog) -> Dict[str, FrozenSet[str]]:
-        sheet_to_column_name = {}
-        for configured_stream in catalog.streams:
-            stream = configured_stream.stream
-            sheet_name = stream.name
-            sheet_to_column_name[sheet_name] = frozenset(stream.json_schema["properties"].keys())
-
-        return sheet_to_column_name
-
-    @staticmethod
-    def row_data_to_record_message(sheet_name: str, cell_values: List[str], column_index_to_name: Dict[int, str]) -> AirbyteRecordMessage:
-        data = {}
-        for relevant_index in sorted(column_index_to_name.keys()):
-            if relevant_index >= len(cell_values):
-                break
-
-            cell_value = cell_values[relevant_index]
-            if cell_value.strip() != "":
-                data[column_index_to_name[relevant_index]] = cell_value
-
-        return AirbyteRecordMessage(stream=sheet_name, data=data, emitted_at=int(datetime.now().timestamp()) * 1000)
-
-    @staticmethod
-    def get_available_sheets_to_column_index_to_name(
-        client, spreadsheet_id: str, requested_sheets_and_columns: Dict[str, FrozenSet[str]], names_conversion: bool = False
-    ) -> Dict[str, Dict[int, str]]:
-        available_sheets = Helpers.get_sheets_in_spreadsheet(client, spreadsheet_id)
-        logger.info(f"Available sheets: {available_sheets}")
-        available_sheets_to_column_index_to_name = defaultdict(dict)
-        for sheet, columns in requested_sheets_and_columns.items():
-            if sheet in available_sheets:
-                first_row = Helpers.get_first_row(client, spreadsheet_id, sheet)
-                if names_conversion:
-                    first_row = [safe_name_conversion(h) for h in first_row]
-                    # When performing names conversion, they won't match what is listed in catalog for the majority of cases,
-                    # so they should be cast here in order to have them in records
-                    columns = {safe_name_conversion(c) for c in columns}
-                # Find the column index of each header value
-                idx = 0
-                for cell_value in first_row:
-                    if cell_value in columns:
-                        available_sheets_to_column_index_to_name[sheet][idx] = cell_value
-                    idx += 1
-        return available_sheets_to_column_index_to_name
-
-    @staticmethod
-    def get_sheets_in_spreadsheet(client, spreadsheet_id: str) -> List[str]:
-        spreadsheet_metadata = Spreadsheet.parse_obj(client.get(spreadsheetId=spreadsheet_id, includeGridData=False))
-        return [sheet.properties.title for sheet in spreadsheet_metadata.sheets]
-
-    @staticmethod
-    def get_sheet_row_count(client, spreadsheet_id: str) -> Dict[str, int]:
-        spreadsheet_metadata = Spreadsheet.parse_obj(client.get(spreadsheetId=spreadsheet_id, includeGridData=False))
-        # filter out sheets without gridProperties (like in diagram sheets)
-        data_sheets = [sheet for sheet in spreadsheet_metadata.sheets if hasattr(sheet.properties, "gridProperties")]
-        return {sheet.properties.title: sheet.properties.gridProperties["rowCount"] for sheet in data_sheets}
-
-    @staticmethod
-    def get_grid_sheets(spreadsheet_metadata) -> List[str]:
-        """Return grid only diagram, filter out sheets with image/diagram only
-
-        https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/sheets#sheetproperties
-        """
-        grid_sheets = []
-        non_grid_sheets = []
-        for sheet in spreadsheet_metadata.sheets:
-            sheet_title = sheet.properties.title
-            if (
-                hasattr(sheet.properties, "gridProperties")
-                and hasattr(sheet.properties, "sheetType")
-                and sheet.properties.sheetType == "GRID"
-            ):
-                grid_sheets.append(sheet_title)
-            else:
-                non_grid_sheets.append(sheet_title)
-
-        if non_grid_sheets:
-            # logging.getLogger(...).log() expects an integer level. The level for WARN is 30
-            # Reference: https://docs.python.org/3.10/library/logging.html#levels
-            logging.getLogger("airbyte").log(30, "Skip non-grid sheets: " + ", ".join(non_grid_sheets))
-
-        return grid_sheets
-
-    @staticmethod
-    def is_row_empty(cell_values: List[str]) -> bool:
-        for cell in cell_values:
-            if cell.strip() != "":
-                return False
-        return True
-
-    @staticmethod
-    def row_contains_relevant_data(cell_values: List[str], relevant_indices: Iterable[int]) -> bool:
-        for idx in relevant_indices:
-            if len(cell_values) > idx and cell_values[idx].strip() != "":
-                return True
-        return False
-
-    @staticmethod
-    def get_spreadsheet_id(id_or_url: str) -> str:
-        if re.match(r"(https://)", id_or_url):
-            # This is a URL
-            m = re.search(r"(/)([-\w]{20,})([/]?)", id_or_url)
-            if m is not None and m.group(2):
-                return m.group(2)
-        else:
-            return id_or_url
-
-    @staticmethod
-    def check_sheet_is_valid(client, spreadsheet_id: str, sheet_name: str) -> Tuple[bool, str]:
-        try:
-            Helpers.get_first_row(client, spreadsheet_id, sheet_name)
-            return True, ""
-        except Exception as e:
-            return False, str(e)