airbyte-cdk 6.20.1__py3-none-any.whl → 6.20.2.dev0__py3-none-any.whl

airbyte_cdk/sources/file_based/file_based_stream_reader.py

@@ -135,17 +135,6 @@ class AbstractFileBasedStreamReader(ABC):
             return use_file_transfer
         return False
 
-    def preserve_directory_structure(self) -> bool:
-        # fall back to preserve subdirectories if config is not present or incomplete
-        if (
-            self.use_file_transfer()
-            and self.config
-            and hasattr(self.config.delivery_method, "preserve_directory_structure")
-            and self.config.delivery_method.preserve_directory_structure is not None
-        ):
-            return self.config.delivery_method.preserve_directory_structure
-        return True
-
     @abstractmethod
     def get_file(
         self, file: RemoteFile, local_directory: str, logger: logging.Logger
@@ -170,13 +159,10 @@ class AbstractFileBasedStreamReader(ABC):
         """
         ...
 
-    def _get_file_transfer_paths(self, file: RemoteFile, local_directory: str) -> List[str]:
-        preserve_directory_structure = self.preserve_directory_structure()
-        if preserve_directory_structure:
-            # Remove left slashes from source path format to make relative path for writing locally
-            file_relative_path = file.uri.lstrip("/")
-        else:
-            file_relative_path = path.basename(file.uri)
+    @staticmethod
+    def _get_file_transfer_paths(file: RemoteFile, local_directory: str) -> List[str]:
+        # Remove left slashes from source path format to make relative path for writing locally
+        file_relative_path = file.uri.lstrip("/")
         local_file_path = path.join(local_directory, file_relative_path)
 
         # Ensure the local directory exists

airbyte_cdk/sources/file_based/file_types/unstructured_parser.py

@@ -2,7 +2,6 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 import logging
-import os
 import traceback
 from datetime import datetime
 from io import BytesIO, IOBase
@@ -43,34 +42,12 @@ unstructured_partition_pdf = None
 unstructured_partition_docx = None
 unstructured_partition_pptx = None
 
-AIRBYTE_NLTK_DATA_DIR = "/airbyte/nltk_data"
-TMP_NLTK_DATA_DIR = "/tmp/nltk_data"
-
-
-def get_nltk_temp_folder() -> str:
-    """
-    For non-root connectors /tmp is not currently writable, but we should allow it in the future.
-    It's safe to use /airbyte for now. Fallback to /tmp for local development.
-    """
-    try:
-        nltk_data_dir = AIRBYTE_NLTK_DATA_DIR
-        os.makedirs(nltk_data_dir, exist_ok=True)
-    except OSError:
-        nltk_data_dir = TMP_NLTK_DATA_DIR
-        os.makedirs(nltk_data_dir, exist_ok=True)
-    return nltk_data_dir
-
-
 try:
-    nltk_data_dir = get_nltk_temp_folder()
-    nltk.data.path.append(nltk_data_dir)
     nltk.data.find("tokenizers/punkt.zip")
     nltk.data.find("tokenizers/punkt_tab.zip")
-    nltk.data.find("tokenizers/averaged_perceptron_tagger_eng.zip")
 except LookupError:
-    nltk.download("punkt", download_dir=nltk_data_dir, quiet=True)
-    nltk.download("punkt_tab", download_dir=nltk_data_dir, quiet=True)
-    nltk.download("averaged_perceptron_tagger_eng", download_dir=nltk_data_dir, quiet=True)
+    nltk.download("punkt")
+    nltk.download("punkt_tab")
 
 
 def optional_decode(contents: Union[str, bytes]) -> str:

airbyte_cdk/sources/file_based/stream/default_file_based_stream.py

@@ -5,17 +5,14 @@
 import asyncio
 import itertools
 import traceback
-from collections import defaultdict
 from copy import deepcopy
 from functools import cache
-from os import path
-from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Set, Tuple, Union
+from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Set, Union
 
 from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, FailureType, Level
 from airbyte_cdk.models import Type as MessageType
 from airbyte_cdk.sources.file_based.config.file_based_stream_config import PrimaryKeyType
 from airbyte_cdk.sources.file_based.exceptions import (
-    DuplicatedFilesError,
     FileBasedSourceError,
     InvalidSchemaError,
     MissingSchemaError,
@@ -46,8 +43,6 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
     """
 
     FILE_TRANSFER_KW = "use_file_transfer"
-    PRESERVE_DIRECTORY_STRUCTURE_KW = "preserve_directory_structure"
-    FILES_KEY = "files"
     DATE_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
     ab_last_mod_col = "_ab_source_file_last_modified"
     ab_file_name_col = "_ab_source_file_url"
@@ -55,15 +50,10 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
     source_file_url = "source_file_url"
     airbyte_columns = [ab_last_mod_col, ab_file_name_col]
     use_file_transfer = False
-    preserve_directory_structure = True
 
     def __init__(self, **kwargs: Any):
         if self.FILE_TRANSFER_KW in kwargs:
             self.use_file_transfer = kwargs.pop(self.FILE_TRANSFER_KW, False)
-        if self.PRESERVE_DIRECTORY_STRUCTURE_KW in kwargs:
-            self.preserve_directory_structure = kwargs.pop(
-                self.PRESERVE_DIRECTORY_STRUCTURE_KW, True
-            )
         super().__init__(**kwargs)
 
     @property
@@ -108,33 +98,15 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
         else:
             return super()._filter_schema_invalid_properties(configured_catalog_json_schema)
 
-    def _duplicated_files_names(
-        self, slices: List[dict[str, List[RemoteFile]]]
-    ) -> List[dict[str, List[str]]]:
-        seen_file_names: Dict[str, List[str]] = defaultdict(list)
-        for file_slice in slices:
-            for file_found in file_slice[self.FILES_KEY]:
-                file_name = path.basename(file_found.uri)
-                seen_file_names[file_name].append(file_found.uri)
-        return [
-            {file_name: paths} for file_name, paths in seen_file_names.items() if len(paths) > 1
-        ]
-
     def compute_slices(self) -> Iterable[Optional[Mapping[str, Any]]]:
         # Sort files by last_modified, uri and return them grouped by last_modified
         all_files = self.list_files()
         files_to_read = self._cursor.get_files_to_sync(all_files, self.logger)
         sorted_files_to_read = sorted(files_to_read, key=lambda f: (f.last_modified, f.uri))
         slices = [
-            {self.FILES_KEY: list(group[1])}
+            {"files": list(group[1])}
             for group in itertools.groupby(sorted_files_to_read, lambda f: f.last_modified)
         ]
-        if slices and not self.preserve_directory_structure:
-            duplicated_files_names = self._duplicated_files_names(slices)
-            if duplicated_files_names:
-                raise DuplicatedFilesError(
-                    stream=self.name, duplicated_files_names=duplicated_files_names
-                )
         return slices
 
     def transform_record(

airbyte_cdk/sources/streams/concurrent/cursor.py

@@ -196,7 +196,9 @@ class ConcurrentCursor(Cursor):
 
     @property
     def state(self) -> MutableMapping[str, Any]:
-        return self._concurrent_state
+        return self._connector_state_converter.convert_to_state_message(
+            self.cursor_field, self._concurrent_state
+        )
 
     @property
     def cursor_field(self) -> CursorField:
@@ -241,10 +243,10 @@ class ConcurrentCursor(Cursor):
         return self._connector_state_converter.parse_value(self._cursor_field.extract_value(record))
 
     def close_partition(self, partition: Partition) -> None:
-        slice_count_before = len(self.state.get("slices", []))
+        slice_count_before = len(self._concurrent_state.get("slices", []))
         self._add_slice_to_state(partition)
         if slice_count_before < len(
-            self.state["slices"]
+            self._concurrent_state["slices"]
         ):  # only emit if at least one slice has been processed
             self._merge_partitions()
             self._emit_state_message()
@@ -256,11 +258,11 @@ class ConcurrentCursor(Cursor):
         )
 
         if self._slice_boundary_fields:
-            if "slices" not in self.state:
+            if "slices" not in self._concurrent_state:
                 raise RuntimeError(
                     f"The state for stream {self._stream_name} should have at least one slice to delineate the sync start time, but no slices are present. This is unexpected. Please contact Support."
                 )
-            self.state["slices"].append(
+            self._concurrent_state["slices"].append(
                 {
                     self._connector_state_converter.START_KEY: self._extract_from_slice(
                         partition, self._slice_boundary_fields[self._START_BOUNDARY]
@@ -288,7 +290,7 @@ class ConcurrentCursor(Cursor):
                     "expected. Please contact the Airbyte team."
                 )
 
-            self.state["slices"].append(
+            self._concurrent_state["slices"].append(
                 {
                     self._connector_state_converter.START_KEY: self.start,
                     self._connector_state_converter.END_KEY: most_recent_cursor_value,
@@ -300,9 +302,7 @@ class ConcurrentCursor(Cursor):
         self._connector_state_manager.update_state_for_stream(
             self._stream_name,
             self._stream_namespace,
-            self._connector_state_converter.convert_to_state_message(
-                self._cursor_field, self.state
-            ),
+            self.state,
         )
         state_message = self._connector_state_manager.create_state_message(
             self._stream_name, self._stream_namespace
@@ -310,7 +310,9 @@ class ConcurrentCursor(Cursor):
         self._message_repository.emit_message(state_message)
 
     def _merge_partitions(self) -> None:
-        self.state["slices"] = self._connector_state_converter.merge_intervals(self.state["slices"])
+        self._concurrent_state["slices"] = self._connector_state_converter.merge_intervals(
+            self._concurrent_state["slices"]
+        )
 
     def _extract_from_slice(self, partition: Partition, key: str) -> CursorValueType:
         try:
@@ -347,36 +349,42 @@ class ConcurrentCursor(Cursor):
         if self._start is not None and self._is_start_before_first_slice():
             yield from self._split_per_slice_range(
                 self._start,
-                self.state["slices"][0][self._connector_state_converter.START_KEY],
+                self._concurrent_state["slices"][0][self._connector_state_converter.START_KEY],
                 False,
             )
 
-        if len(self.state["slices"]) == 1:
+        if len(self._concurrent_state["slices"]) == 1:
             yield from self._split_per_slice_range(
                 self._calculate_lower_boundary_of_last_slice(
-                    self.state["slices"][0][self._connector_state_converter.END_KEY]
+                    self._concurrent_state["slices"][0][self._connector_state_converter.END_KEY]
                 ),
                 self._end_provider(),
                 True,
             )
-        elif len(self.state["slices"]) > 1:
-            for i in range(len(self.state["slices"]) - 1):
+        elif len(self._concurrent_state["slices"]) > 1:
+            for i in range(len(self._concurrent_state["slices"]) - 1):
                 if self._cursor_granularity:
                     yield from self._split_per_slice_range(
-                        self.state["slices"][i][self._connector_state_converter.END_KEY]
+                        self._concurrent_state["slices"][i][self._connector_state_converter.END_KEY]
                         + self._cursor_granularity,
-                        self.state["slices"][i + 1][self._connector_state_converter.START_KEY],
+                        self._concurrent_state["slices"][i + 1][
+                            self._connector_state_converter.START_KEY
+                        ],
                         False,
                     )
                 else:
                     yield from self._split_per_slice_range(
-                        self.state["slices"][i][self._connector_state_converter.END_KEY],
-                        self.state["slices"][i + 1][self._connector_state_converter.START_KEY],
+                        self._concurrent_state["slices"][i][
+                            self._connector_state_converter.END_KEY
+                        ],
+                        self._concurrent_state["slices"][i + 1][
+                            self._connector_state_converter.START_KEY
+                        ],
                         False,
                     )
             yield from self._split_per_slice_range(
                 self._calculate_lower_boundary_of_last_slice(
-                    self.state["slices"][-1][self._connector_state_converter.END_KEY]
+                    self._concurrent_state["slices"][-1][self._connector_state_converter.END_KEY]
                 ),
                 self._end_provider(),
                 True,
@@ -387,7 +395,8 @@ class ConcurrentCursor(Cursor):
     def _is_start_before_first_slice(self) -> bool:
         return (
             self._start is not None
-            and self._start < self.state["slices"][0][self._connector_state_converter.START_KEY]
+            and self._start
+            < self._concurrent_state["slices"][0][self._connector_state_converter.START_KEY]
         )
 
     def _calculate_lower_boundary_of_last_slice(

airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py

@@ -81,10 +81,10 @@ class AbstractOauth2Authenticator(AuthBase):
         Override to define additional parameters
         """
         payload: MutableMapping[str, Any] = {
-            self.get_grant_type_name(): self.get_grant_type(),
-            self.get_client_id_name(): self.get_client_id(),
-            self.get_client_secret_name(): self.get_client_secret(),
-            self.get_refresh_token_name(): self.get_refresh_token(),
+            "grant_type": self.get_grant_type(),
+            "client_id": self.get_client_id(),
+            "client_secret": self.get_client_secret(),
+            "refresh_token": self.get_refresh_token(),
         }
 
         if self.get_scopes():
@@ -98,14 +98,6 @@ class AbstractOauth2Authenticator(AuthBase):
 
         return payload
 
-    def build_refresh_request_headers(self) -> Mapping[str, Any] | None:
-        """
-        Returns the request headers to set on the refresh request
-
-        """
-        headers = self.get_refresh_request_headers()
-        return headers if headers else None
-
     def _wrap_refresh_token_exception(
         self, exception: requests.exceptions.RequestException
     ) -> bool:
@@ -136,7 +128,6 @@ class AbstractOauth2Authenticator(AuthBase):
                 method="POST",
                 url=self.get_token_refresh_endpoint(),  # type: ignore # returns None, if not provided, but str | bytes is expected.
                 data=self.build_refresh_request_body(),
-                headers=self.build_refresh_request_headers(),
             )
             if response.ok:
                 response_json = response.json()
@@ -215,26 +206,14 @@ class AbstractOauth2Authenticator(AuthBase):
     def get_token_refresh_endpoint(self) -> Optional[str]:
         """Returns the endpoint to refresh the access token"""
 
-    @abstractmethod
-    def get_client_id_name(self) -> str:
-        """The client id name to authenticate"""
-
     @abstractmethod
     def get_client_id(self) -> str:
         """The client id to authenticate"""
 
-    @abstractmethod
-    def get_client_secret_name(self) -> str:
-        """The client secret name to authenticate"""
-
     @abstractmethod
     def get_client_secret(self) -> str:
         """The client secret to authenticate"""
 
-    @abstractmethod
-    def get_refresh_token_name(self) -> str:
-        """The refresh token name to authenticate"""
-
     @abstractmethod
     def get_refresh_token(self) -> Optional[str]:
         """The token used to refresh the access token when it expires"""
@@ -263,18 +242,10 @@ class AbstractOauth2Authenticator(AuthBase):
     def get_refresh_request_body(self) -> Mapping[str, Any]:
         """Returns the request body to set on the refresh request"""
 
-    @abstractmethod
-    def get_refresh_request_headers(self) -> Mapping[str, Any]:
-        """Returns the request headers to set on the refresh request"""
-
     @abstractmethod
     def get_grant_type(self) -> str:
         """Returns grant_type specified for requesting access_token"""
 
-    @abstractmethod
-    def get_grant_type_name(self) -> str:
-        """Returns grant_type specified name for requesting access_token"""
-
     @property
     @abstractmethod
     def access_token(self) -> str:

airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py

@@ -30,17 +30,12 @@ class Oauth2Authenticator(AbstractOauth2Authenticator):
         client_id: str,
         client_secret: str,
         refresh_token: str,
-        client_id_name: str = "client_id",
-        client_secret_name: str = "client_secret",
-        refresh_token_name: str = "refresh_token",
         scopes: List[str] | None = None,
         token_expiry_date: pendulum.DateTime | None = None,
         token_expiry_date_format: str | None = None,
         access_token_name: str = "access_token",
         expires_in_name: str = "expires_in",
         refresh_request_body: Mapping[str, Any] | None = None,
-        refresh_request_headers: Mapping[str, Any] | None = None,
-        grant_type_name: str = "grant_type",
         grant_type: str = "refresh_token",
         token_expiry_is_time_of_expiration: bool = False,
         refresh_token_error_status_codes: Tuple[int, ...] = (),
@@ -48,18 +43,13 @@ class Oauth2Authenticator(AbstractOauth2Authenticator):
         refresh_token_error_values: Tuple[str, ...] = (),
     ):
         self._token_refresh_endpoint = token_refresh_endpoint
-        self._client_secret_name = client_secret_name
         self._client_secret = client_secret
-        self._client_id_name = client_id_name
         self._client_id = client_id
-        self._refresh_token_name = refresh_token_name
         self._refresh_token = refresh_token
         self._scopes = scopes
         self._access_token_name = access_token_name
         self._expires_in_name = expires_in_name
         self._refresh_request_body = refresh_request_body
-        self._refresh_request_headers = refresh_request_headers
-        self._grant_type_name = grant_type_name
         self._grant_type = grant_type
 
         self._token_expiry_date = token_expiry_date or pendulum.now().subtract(days=1)  # type: ignore [no-untyped-call]
@@ -73,21 +63,12 @@ class Oauth2Authenticator(AbstractOauth2Authenticator):
     def get_token_refresh_endpoint(self) -> str:
         return self._token_refresh_endpoint
 
-    def get_client_id_name(self) -> str:
-        return self._client_id_name
-
     def get_client_id(self) -> str:
         return self._client_id
 
-    def get_client_secret_name(self) -> str:
-        return self._client_secret_name
-
     def get_client_secret(self) -> str:
         return self._client_secret
 
-    def get_refresh_token_name(self) -> str:
-        return self._refresh_token_name
-
     def get_refresh_token(self) -> str:
         return self._refresh_token
 
@@ -103,12 +84,6 @@ class Oauth2Authenticator(AbstractOauth2Authenticator):
     def get_refresh_request_body(self) -> Mapping[str, Any]:
         return self._refresh_request_body  # type: ignore [return-value]
 
-    def get_refresh_request_headers(self) -> Mapping[str, Any]:
-        return self._refresh_request_headers  # type: ignore [return-value]
-
-    def get_grant_type_name(self) -> str:
-        return self._grant_type_name
-
     def get_grant_type(self) -> str:
         return self._grant_type
 
@@ -154,12 +129,8 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
         expires_in_name: str = "expires_in",
         refresh_token_name: str = "refresh_token",
         refresh_request_body: Mapping[str, Any] | None = None,
-        refresh_request_headers: Mapping[str, Any] | None = None,
-        grant_type_name: str = "grant_type",
         grant_type: str = "refresh_token",
-        client_id_name: str = "client_id",
         client_id: Optional[str] = None,
-        client_secret_name: str = "client_secret",
         client_secret: Optional[str] = None,
         access_token_config_path: Sequence[str] = ("credentials", "access_token"),
         refresh_token_config_path: Sequence[str] = ("credentials", "refresh_token"),
@@ -180,7 +151,6 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
             expires_in_name (str, optional): Name of the name of the field that characterizes when the current access token will expire, used to parse the refresh token response. Defaults to "expires_in".
             refresh_token_name (str, optional): Name of the name of the refresh token field, used to parse the refresh token response. Defaults to "refresh_token".
             refresh_request_body (Mapping[str, Any], optional): Custom key value pair that will be added to the refresh token request body. Defaults to None.
-            refresh_request_headers (Mapping[str, Any], optional): Custom key value pair that will be added to the refresh token request headers. Defaults to None.
             grant_type (str, optional): OAuth grant type. Defaults to "refresh_token".
             client_id (Optional[str]): The client id to authenticate. If not specified, defaults to credentials.client_id in the config object.
             client_secret (Optional[str]): The client secret to authenticate. If not specified, defaults to credentials.client_secret in the config object.
@@ -204,31 +174,23 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
                 ("credentials", "client_secret"),
             )
         )
-        self._client_id_name = client_id_name
-        self._client_secret_name = client_secret_name
         self._access_token_config_path = access_token_config_path
         self._refresh_token_config_path = refresh_token_config_path
         self._token_expiry_date_config_path = token_expiry_date_config_path
         self._token_expiry_date_format = token_expiry_date_format
         self._refresh_token_name = refresh_token_name
-        self._grant_type_name = grant_type_name
         self._connector_config = connector_config
         self.__message_repository = message_repository
         super().__init__(
-            token_refresh_endpoint=token_refresh_endpoint,
-            client_id_name=self._client_id_name,
-            client_id=self.get_client_id(),
-            client_secret_name=self._client_secret_name,
-            client_secret=self.get_client_secret(),
-            refresh_token=self.get_refresh_token(),
-            refresh_token_name=self._refresh_token_name,
+            token_refresh_endpoint,
+            self.get_client_id(),
+            self.get_client_secret(),
+            self.get_refresh_token(),
             scopes=scopes,
             token_expiry_date=self.get_token_expiry_date(),
             access_token_name=access_token_name,
             expires_in_name=expires_in_name,
             refresh_request_body=refresh_request_body,
-            refresh_request_headers=refresh_request_headers,
-            grant_type_name=self._grant_type_name,
             grant_type=grant_type,
             token_expiry_date_format=token_expiry_date_format,
             token_expiry_is_time_of_expiration=token_expiry_is_time_of_expiration,

airbyte_cdk/sources/types.py

@@ -152,6 +152,3 @@ class StreamSlice(Mapping[str, Any]):
 
     def __hash__(self) -> int:
         return hash(orjson.dumps(self._stream_slice, option=orjson.OPT_SORT_KEYS))
-
-    def __bool__(self) -> bool:
-        return bool(self._stream_slice) or bool(self._extra_fields)

{airbyte_cdk-6.20.1.dist-info → airbyte_cdk-6.20.2.dev0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: airbyte-cdk
-Version: 6.20.1
+Version: 6.20.2.dev0
 Summary: A framework for writing Airbyte Connectors.
 License: MIT
 Keywords: airbyte,connector-development-kit,cdk