airbyte-source-google-sheets 0.10.0.dev202505271949__py3-none-any.whl → 0.11.0__py3-none-any.whl

{airbyte_source_google_sheets-0.10.0.dev202505271949.dist-info → airbyte_source_google_sheets-0.11.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: airbyte-source-google-sheets
-Version: 0.10.0.dev202505271949
+Version: 0.11.0
 Summary: Source implementation for Google Sheets.
 License: Elv2
 Author: Airbyte

airbyte_source_google_sheets-0.11.0.dist-info/RECORD

@@ -0,0 +1,16 @@
+source_google_sheets/__init__.py,sha256=dYaZS0KrTjksk_yeSuXmNDXDsNKWctFnlZJSMgLryXE,135
+source_google_sheets/components/__init__.py,sha256=v7odPaLdz2S_NRzExsxkk6694Vmjbplz4Z5mA4lxdkA,343
+source_google_sheets/components/extractors.py,sha256=7CLiYFYTq6DfeYCxchvQI_sQLpaVd_86Zfon2hKZe9Q,9842
+source_google_sheets/components/partition_routers.py,sha256=SWo1V0K10ZdWE2TQ0KuQUfue04RTyHJe1f6BOj6c96s,1265
+source_google_sheets/manifest.yaml,sha256=GHQb_x_2pZQR4Oh5gWMnAm-KEGvuZUENYoX4PoUUR7Y,16489
+source_google_sheets/models/__init__.py,sha256=Z-4MTpxG5t2jGhXzs4PPoIOa83zw3jRnUDx0N9Puv3s,61
+source_google_sheets/models/spreadsheet.py,sha256=DEef7bWQEpY1Uqyk7RN6qLF8oxLSNzuAtHlGRmMAKQA,1112
+source_google_sheets/models/spreadsheet_values.py,sha256=-XRMuuILn9JN8svHNTj6-oG8mLTZOZ5Hejy2pJ5bILk,440
+source_google_sheets/run.py,sha256=eaPRcarWqkB2b2DokvI83w7rz1blmWPQCFahvCyCdSY,1887
+source_google_sheets/source.py,sha256=qO1KoGdphieu7F5VgDYtrbqs56AUvMWFGNvFHP2b9Z4,778
+source_google_sheets/spec.yaml,sha256=RN3XrLj6PYzqTNtpDxYo6PffkyPPizEBOxSzCrPL15Q,9058
+source_google_sheets/utils.py,sha256=jiVPqsRDjVgdwIiBJMvFJEwwuUBQ7BQAebRqfpS9pZw,6943
+airbyte_source_google_sheets-0.11.0.dist-info/METADATA,sha256=VBrcsENgMSKx9zxYAfcmDh03ezpuH1dCRbA2zezrxG0,5369
+airbyte_source_google_sheets-0.11.0.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+airbyte_source_google_sheets-0.11.0.dist-info/entry_points.txt,sha256=Dtsfjohe5IPUFyqojk49SIoP7CifCTlNLG_pgivzppo,69
+airbyte_source_google_sheets-0.11.0.dist-info/RECORD,,

source_google_sheets/components/extractors.py

@@ -12,7 +12,10 @@ from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder
 from airbyte_cdk.sources.declarative.extractors.dpath_extractor import DpathExtractor
 from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
 from airbyte_cdk.sources.types import Config
-from source_google_sheets.utils import experimental_safe_name_conversion, name_conversion, safe_name_conversion
+from source_google_sheets.utils import (
+    safe_name_conversion,
+    safe_sanitzation_conversion,
+)
 
 
 class RawSchemaParser:
@@ -54,7 +57,6 @@ class RawSchemaParser:
         schema_pointer: List[Union[InterpolatedString, str]],
         key_pointer: List[Union[InterpolatedString, str]],
         names_conversion: bool,
-        experimental_names_conversion: bool,
     ):
         """
         1. Parses sheet headers from the provided raw schema. This method assumes that data is contiguous
@@ -68,13 +70,24 @@ class RawSchemaParser:
         duplicate_fields = set()
         parsed_schema_values = []
         seen_values = set()
+        # Gather all sanitisation flags from config
+        config = getattr(self, "config", {})
+        flags = {
+            "remove_leading_trailing_underscores": config.get("remove_leading_trailing_underscores", False),
+            "combine_number_word_pairs": config.get("combine_number_word_pairs", False),
+            "remove_special_characters": config.get("remove_special_characters", False),
+            "combine_letter_number_pairs": config.get("combine_letter_number_pairs", False),
+            "allow_leading_numbers": config.get("allow_leading_numbers", False),
+        }
+        use_sanitzation = any(flags.values())
+
         for property_index, raw_schema_property in enumerate(raw_schema_properties):
             raw_schema_property_value = self._extract_data(raw_schema_property, key_pointer)
             if not raw_schema_property_value or raw_schema_property_value.isspace():
                 break
-            # Apply experimental conversion if enabled; otherwise, apply standard conversion if enabled
-            if experimental_names_conversion:
-                raw_schema_property_value = experimental_safe_name_conversion(raw_schema_property_value)
+            # Use sanitzation if any flag is set, else legacy
+            if names_conversion and use_sanitzation:
+                raw_schema_property_value = safe_sanitzation_conversion(raw_schema_property_value, **flags)
             elif names_conversion:
                 raw_schema_property_value = safe_name_conversion(raw_schema_property_value)
 
@@ -93,13 +106,12 @@ class RawSchemaParser:
     def parse(self, schema_type_identifier, records: Iterable[MutableMapping[Any, Any]]):
         """Removes duplicated fields and makes names conversion"""
         names_conversion = self.config.get("names_conversion", False)
-        experimental_names_conversion = self.config.get("experimental_names_conversion", False)
         schema_pointer = schema_type_identifier.get("schema_pointer")
         key_pointer = schema_type_identifier["key_pointer"]
         parsed_properties = []
         for raw_schema_data in records:
             for _, parsed_value, raw_schema_property in self.parse_raw_schema_values(
-                raw_schema_data, schema_pointer, key_pointer, names_conversion, experimental_names_conversion
+                raw_schema_data, schema_pointer, key_pointer, names_conversion
             ):
                 self._set_data(parsed_value, raw_schema_property, key_pointer)
                 parsed_properties.append(raw_schema_property)
@@ -145,20 +157,16 @@ class DpathSchemaMatchingExtractor(DpathExtractor, RawSchemaParser):
         self._values_to_match_key = parameters["values_to_match_key"]
         schema_type_identifier = parameters["schema_type_identifier"]
         names_conversion = self.config.get("names_conversion", False)
-        experimental_names_conversion = self.config.get("experimental_names_conversion", False)
         self._indexed_properties_to_match = self.extract_properties_to_match(
-            parameters["properties_to_match"],
-            schema_type_identifier,
-            names_conversion=names_conversion,
-            experimental_names_conversion=experimental_names_conversion,
+            parameters["properties_to_match"], schema_type_identifier, names_conversion=names_conversion
         )
 
-    def extract_properties_to_match(self, properties_to_match, schema_type_identifier, names_conversion, experimental_names_conversion):
+    def extract_properties_to_match(self, properties_to_match, schema_type_identifier, names_conversion):
         schema_pointer = schema_type_identifier.get("schema_pointer")
         key_pointer = schema_type_identifier["key_pointer"]
         indexed_properties = {}
         for property_index, property_parsed_value, _ in self.parse_raw_schema_values(
-            properties_to_match, schema_pointer, key_pointer, names_conversion, experimental_names_conversion
+            properties_to_match, schema_pointer, key_pointer, names_conversion
         ):
             indexed_properties[property_index] = property_parsed_value
         return indexed_properties
@@ -202,6 +210,7 @@ class DpathSchemaMatchingExtractor(DpathExtractor, RawSchemaParser):
                     )
 
 
+@dataclass
 class DpathSchemaExtractor(DpathExtractor, RawSchemaParser):
     """
     Makes names conversion and parses sheet headers from the provided row.

source_google_sheets/manifest.yaml

@@ -105,9 +105,16 @@ dynamic_streams:
         - field_path:
             - name
           type: ComponentMappingDefinition
-          value: "{{components_values['properties']['title']}}"
+          # Expression is structured as: <value_if_true> if <condition> else <value_if_false>
+          # This sets the stream name to the custom name from stream_name_overrides if a match is found,
+          # otherwise uses the original sheet/tab name.
+          value: >-
+            {%- set overrides = config.get('stream_name_overrides', [])
+                | selectattr('source_stream_name', 'equalto', components_values['properties']['title'])
+                | list -%}
+            {{ overrides[0]['custom_stream_name'] if overrides else components_values['properties']['title'] }}
           value_type: string
-          description: name for dynamic stream.
+          description: name for dynamic stream (with optional override).
         - field_path:
             - schema_loader
             - retriever

source_google_sheets/spec.yaml

@@ -21,6 +21,7 @@ connectionSpecification:
         otherwise the request returns a timeout error. In regards to this information, consider network speed and
         number of columns of the google sheet when deciding a batch_size value.
       default: 1000000
+      order: 1
     spreadsheet_id:
       type: string
       title: Spreadsheet Link
@@ -28,19 +29,60 @@ connectionSpecification:
         Enter the link to the Google spreadsheet you want to sync. To copy the link, click the 'Share' button in the top-right corner of the spreadsheet, then click 'Copy link'.
       examples:
         - https://docs.google.com/spreadsheets/d/1hLd9Qqti3UyLXZB2aFfUWDT7BG-arw2xy4HR3D-dwUb/edit
+      order: 0
     names_conversion:
       type: boolean
       title: Convert Column Names to SQL-Compliant Format
-      description: Enables the conversion of column names to a standardized, SQL-compliant format. For example, 'My Name' -> 'my_name'. Enable this option if your destination is SQL-based.
+      description: >-
+        Converts column names to a SQL-compliant format (snake_case, lowercase, etc).
+        If enabled, you can further customize the sanitization using the options below.
+      default: false
+      order: 2
+    remove_leading_trailing_underscores:
+      type: boolean
+      title: Remove Leading and Trailing Underscores
+      description: >-
+        Removes leading and trailing underscores from column names. Does not remove leading underscores from column names that start with a number.
+        Example: "50th Percentile? "→ "_50_th_percentile"
+        This option will only work if "Convert Column Names to SQL-Compliant Format (names_conversion)" is enabled.
       default: false
-    experimental_names_conversion:
+      order: 3
+    combine_number_word_pairs:
       type: boolean
-      title: Experimental Convert Column Names to SQL-Compliant Format
+      title: Combine Number-Word Pairs
       description: >-
-        Adds additional sanitization to column names before converting to SQL-compliant format, such as removing leading and trailing spaces.
-        This option may change behavior in the future, which may cause column names to update in your destination on future updates.
-        Due to this, it is recommended that you also change the "Detect and propagate schema changes" to "Approve all changes myself" in the connection advances settings.
-        If enabled, this option will supersede the `Convert Column Names to SQL-Compliant Format` option.
+        Combines adjacent numbers and words.
+        Example: "50th Percentile?" → "_50th_percentile_"
+        This option will only work if "Convert Column Names to SQL-Compliant Format (names_conversion)" is enabled.
+      default: false
+      order: 4
+    remove_special_characters:
+      type: boolean
+      title: Remove All Special Characters
+      description: >-
+        Removes all special characters from column names.
+        Example: "Example ID*" → "example_id"
+        This option will only work if "Convert Column Names to SQL-Compliant Format (names_conversion)" is enabled.
+      default: false
+      order: 5
+    combine_letter_number_pairs:
+      type: boolean
+      title: Combine Letter-Number Pairs
+      description: >-
+        Combines adjacent letters and numbers.
+        Example: "Q3 2023" → "q3_2023"
+        This option will only work if "Convert Column Names to SQL-Compliant Format (names_conversion)" is enabled.
+      default: false
+      order: 6
+    allow_leading_numbers:
+      type: boolean
+      title: Allow Leading Numbers
+      description: >-
+        Allows column names to start with numbers.
+        Example: "50th Percentile" → "50_th_percentile"
+        This option will only work if "Convert Column Names to SQL-Compliant Format (names_conversion)" is enabled.
+      default: false
+      order: 7
     credentials:
       type: object
       title: Authentication
@@ -89,6 +131,46 @@ connectionSpecification:
               airbyte_secret: true
               examples:
                 - '{ "type": "service_account", "project_id": YOUR_PROJECT_ID, "private_key_id": YOUR_PRIVATE_KEY, ... }'
+    stream_name_overrides:
+      type: array
+      title: Stream Name Overrides
+      description: >-
+        **Overridden streams will default to Sync Mode: Full Refresh (Append), which does not support primary keys. If you want to use primary keys and deduplication, update the sync mode to "Full Refresh | Overwrite + Deduped" in your connection settings.**
+
+        Allows you to rename streams (Google Sheet tab names) as they appear in Airbyte. 
+
+        Each item should be an object with a `source_stream_name` (the exact name of the sheet/tab in your spreadsheet) 
+        and a `custom_stream_name` (the name you want it to appear as in Airbyte and the destination).
+
+        If a `source_stream_name` is not found in your spreadsheet, it will be ignored and the default name will be used.
+        This feature only affects stream (sheet/tab) names, not field/column names.
+
+        If you want to rename fields or column names, you can do so using the Airbyte Mappings feature after your connection is created. See the Airbyte documentation for more details on how to use Mappings.
+
+        Examples:
+          - To rename a sheet called "Sheet1" to "sales_data", and "2024 Q1" to "q1_2024":
+            [
+              { "source_stream_name": "Sheet1", "custom_stream_name": "sales_data" },
+              { "source_stream_name": "2024 Q1", "custom_stream_name": "q1_2024" }
+            ]
+          - If you do not wish to rename any streams, leave this blank.
+      items:
+        type: object
+        required:
+          - source_stream_name
+          - custom_stream_name
+        properties:
+          source_stream_name:
+            type: string
+            title: Source Stream Name
+            description: The exact name of the sheet/tab in your Google Spreadsheet.
+            order: 0
+          custom_stream_name:
+            type: string
+            title: Custom Stream Name
+            description: The name you want this stream to appear as in Airbyte and your destination.
+            order: 1
+        order: 8
 advanced_auth:
   auth_flow_type: oauth2.0
   predicate_key:

source_google_sheets/utils.py

@@ -37,69 +37,126 @@ def name_conversion(text: str) -> str:
     return text
 
 
-def experimental_name_conversion(text: str) -> str:
+def safe_name_conversion(text: str) -> str:
+    if not text:
+        return text
+    new = name_conversion(text)
+    if not new:
+        raise Exception(f"initial string '{text}' converted to empty")
+    return new
+
+
+def _sanitization(
+    text: str,
+    remove_leading_trailing_underscores: bool = False,
+    combine_number_word_pairs: bool = False,
+    remove_special_characters: bool = False,
+    combine_letter_number_pairs: bool = False,
+    allow_leading_numbers: bool = False,
+) -> str:
     """
-    Convert name using a set of rules, for example: '1MyName' -> '_1_my_name'
-    Removes leading/trailing spaces, combines number-word pairs (e.g., '50th' -> '50th'),
-    letter-number pairs (e.g., 'Q3' -> 'Q3'), and removes special characters without adding underscores.
-    Spaces are converted to underscores for snake_case. Preserves spaces between numbers and words.
+    Converts a string into a normalized, SQL-compliant name using a set of configurable options.
+
+    Args:
+        text: The input string to convert.
+        remove_leading_trailing_underscores: If True, removes underscores at the start/end of the result.
+        combine_number_word_pairs: If True, combines adjacent number and word tokens (e.g., "50 th" -> "50th").
+        remove_special_characters: If True, removes all special characters from the input.
+        combine_letter_number_pairs: If True, combines adjacent letter and number tokens (e.g., "Q 3" -> "Q3").
+        allow_leading_numbers: If False, prepends an underscore if the result starts with a number.
+
+    Returns:
+        The normalized, SQL-compliant string.
+
+    Steps:
+    1. Transliterates the input text to ASCII using unidecode.
+    2. Optionally removes special characters if remove_special_characters is True.
+    3. Splits the text into tokens using a regex pattern that separates words, numbers, and non-alphanumeric characters.
+    4. Optionally combines adjacent letter+number or number+word tokens based on flags.
+    5. Removes empty tokens in the middle, but keeps leading/trailing empty tokens for underscore placement.
+    6. Optionally strips leading/trailing underscores if remove_leading_trailing_underscores is True.
+    7. Optionally prepends an underscore if the result starts with a number and allow_leading_numbers is False.
+    8. Returns the final string in lowercase.
     """
-    # Step 1: Tokenization
+    text = unidecode.unidecode(text)
+
+    if remove_special_characters:
+        text = re.sub(r"[^\w\s]", "", text)
+
     tokens = []
     for m in TOKEN_PATTERN.finditer(text):
         if m.group("NoToken") is None:
             tokens.append(m.group(0))
         else:
-            # Process each character in NoToken match
-            for char in m.group(0):
-                if char.isspace():
-                    tokens.append("")
+            tokens.append("")
 
-    # Step 2: Combine adjacent tokens where appropriate
+    # Combine tokens as per flags
     combined_tokens = []
     i = 0
     while i < len(tokens):
-        if i + 1 < len(tokens) and tokens[i] and len(tokens[i]) == 1 and tokens[i].isupper() and tokens[i + 1] and tokens[i + 1].isdigit():
-            combined_tokens.append(tokens[i] + tokens[i + 1])  # e.g., "Q3"
+        if (
+            combine_letter_number_pairs
+            and i + 1 < len(tokens)
+            and tokens[i]
+            and tokens[i].isalpha()
+            and tokens[i + 1]
+            and tokens[i + 1].isdigit()
+        ):
+            combined = tokens[i] + tokens[i + 1]
+            combined_tokens.append(combined)
             i += 2
-        elif i + 1 < len(tokens) and tokens[i] and tokens[i].isdigit() and tokens[i + 1] and tokens[i + 1].isalpha():
-            combined_tokens.append(tokens[i] + tokens[i + 1])  # e.g., "80th"
+        elif (
+            combine_number_word_pairs
+            and i + 1 < len(tokens)
+            and tokens[i]
+            and tokens[i].isdigit()
+            and tokens[i + 1]
+            and tokens[i + 1].isalpha()
+        ):
+            combined = tokens[i] + tokens[i + 1]
+            combined_tokens.append(combined)
             i += 2
         else:
             combined_tokens.append(tokens[i])
             i += 1
 
-    # Step 3: Clean up empty tokens
-    while combined_tokens and combined_tokens[0] == "":
-        combined_tokens.pop(0)
-    while combined_tokens and combined_tokens[-1] == "":
-        combined_tokens.pop()
-    if len(combined_tokens) >= 3:
-        combined_tokens = combined_tokens[:1] + [t for t in combined_tokens[1:-1] if t] + combined_tokens[-1:]
+    # Find indices of first and last non-empty tokens
+    first_non_empty = next((i for i, t in enumerate(combined_tokens) if t), len(combined_tokens))
+    last_non_empty = next((i for i, t in reversed(list(enumerate(combined_tokens))) if t), -1)
 
-    # Step 4: Handle leading digits
-    if combined_tokens and combined_tokens[0].isdigit():
-        combined_tokens.insert(0, "")
+    # Process tokens: keep leading/trailing empty tokens, remove empty tokens in middle
+    if first_non_empty < len(combined_tokens):
+        leading = combined_tokens[:first_non_empty]
+        middle = [t for t in combined_tokens[first_non_empty : last_non_empty + 1] if t]
+        trailing = combined_tokens[last_non_empty + 1 :]
+        processed_tokens = leading + middle + trailing
+    else:
+        processed_tokens = combined_tokens  # All tokens are empty
 
-    # Step 5: Join and convert to lowercase
-    result = DEFAULT_SEPARATOR.join(combined_tokens)
-    return result.lower()
+    # Join tokens with underscores
+    result = DEFAULT_SEPARATOR.join(processed_tokens)
 
+    # Apply remove_leading_trailing_underscores on the final string
+    if remove_leading_trailing_underscores:
+        result = result.strip(DEFAULT_SEPARATOR)
 
-def safe_name_conversion(text: str) -> str:
-    if not text:
-        return text
-    new = name_conversion(text)
-    if not new:
-        raise Exception(f"initial string '{text}' converted to empty")
-    return new
+    # Handle leading numbers after underscore removal
+    if not allow_leading_numbers and result and result[0].isdigit():
+        result = DEFAULT_SEPARATOR + result
 
+    final_result = result.lower()
+    return final_result
 
-def experimental_safe_name_conversion(text: str) -> str:
-    if not text:
-        return text
-    new = experimental_name_conversion(text)
-    if not new:
+
+def safe_sanitzation_conversion(text: str, **kwargs) -> str:
+    """
+    Converts text to a safe name using _sanitization with the provided keyword arguments.
+    Raises an exception if the result is empty or "_". Unlike safe_name_conversion,
+    this function also rejects "_" as a valid result, since _sanitization
+    may return "_" for certain inputs (e.g., "*").
+    """
+    new = _sanitization(text, **kwargs)
+    if not new or new == "_":
         raise Exception(f"initial string '{text}' converted to empty")
     return new
 

airbyte_source_google_sheets-0.10.0.dev202505271949.dist-info/RECORD

@@ -1,16 +0,0 @@
-source_google_sheets/__init__.py,sha256=dYaZS0KrTjksk_yeSuXmNDXDsNKWctFnlZJSMgLryXE,135
-source_google_sheets/components/__init__.py,sha256=v7odPaLdz2S_NRzExsxkk6694Vmjbplz4Z5mA4lxdkA,343
-source_google_sheets/components/extractors.py,sha256=-d5hPW40ZBjU3gTUZNudayvqf1eHlveW1amwKJywDkU,9668
-source_google_sheets/components/partition_routers.py,sha256=SWo1V0K10ZdWE2TQ0KuQUfue04RTyHJe1f6BOj6c96s,1265
-source_google_sheets/manifest.yaml,sha256=CuSnA8dnRMeXWfyUA6aXBvGU3mz1dJIi0HqMiks9Fd0,15938
-source_google_sheets/models/__init__.py,sha256=Z-4MTpxG5t2jGhXzs4PPoIOa83zw3jRnUDx0N9Puv3s,61
-source_google_sheets/models/spreadsheet.py,sha256=DEef7bWQEpY1Uqyk7RN6qLF8oxLSNzuAtHlGRmMAKQA,1112
-source_google_sheets/models/spreadsheet_values.py,sha256=-XRMuuILn9JN8svHNTj6-oG8mLTZOZ5Hejy2pJ5bILk,440
-source_google_sheets/run.py,sha256=eaPRcarWqkB2b2DokvI83w7rz1blmWPQCFahvCyCdSY,1887
-source_google_sheets/source.py,sha256=qO1KoGdphieu7F5VgDYtrbqs56AUvMWFGNvFHP2b9Z4,778
-source_google_sheets/spec.yaml,sha256=mFOiMN1IsrjyLRcjTRw7xa0b74LwkUXdVvUqau7iTXc,5709
-source_google_sheets/utils.py,sha256=eOPrfTFBn7p80gkw4ZEQxskIrOFDj5_Fv9pCGQt4gkA,4576
-airbyte_source_google_sheets-0.10.0.dev202505271949.dist-info/METADATA,sha256=VwpBWod_jy9XLMVHO_oJrDbgGaovtT-1EYEmuRHiq4E,5385
-airbyte_source_google_sheets-0.10.0.dev202505271949.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-airbyte_source_google_sheets-0.10.0.dev202505271949.dist-info/entry_points.txt,sha256=Dtsfjohe5IPUFyqojk49SIoP7CifCTlNLG_pgivzppo,69
-airbyte_source_google_sheets-0.10.0.dev202505271949.dist-info/RECORD,,