airbyte-cdk 6.45.0.dev4107__py3-none-any.whl → 6.45.0.post6.dev14369631849__py3-none-any.whl

airbyte_cdk/sources/declarative/transformations/add_fields.py

@@ -139,7 +139,9 @@ class AddFields(RecordTransformation):
             valid_types = (parsed_field.value_type,) if parsed_field.value_type else None
             value = parsed_field.value.eval(config, valid_types=valid_types, **kwargs)
             is_empty_condition = not self.condition
-            if is_empty_condition or self._filter_interpolator.eval(config, value=value, **kwargs):
+            if is_empty_condition or self._filter_interpolator.eval(
+                config, value=value, path=parsed_field.path, **kwargs
+            ):
                 dpath.new(record, parsed_field.path, value)
 
     def __eq__(self, other: Any) -> bool:

airbyte_cdk/sources/file_based/file_based_stream_reader.py

@@ -8,18 +8,16 @@ from datetime import datetime
 from enum import Enum
 from io import IOBase
 from os import makedirs, path
-from typing import Any, Callable, Iterable, List, MutableMapping, Optional, Set, Tuple
+from typing import Any, Dict, Iterable, List, Optional, Set
 
 from wcmatch.glob import GLOBSTAR, globmatch
 
-from airbyte_cdk.models import AirbyteRecordMessageFileReference
 from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec
 from airbyte_cdk.sources.file_based.config.validate_config_transfer_modes import (
     include_identities_stream,
     preserve_directory_structure,
     use_file_transfer,
 )
-from airbyte_cdk.sources.file_based.file_record_data import FileRecordData
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
 
 
@@ -30,11 +28,6 @@ class FileReadMode(Enum):
 
 class AbstractFileBasedStreamReader(ABC):
     DATE_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
-    FILE_RELATIVE_PATH = "file_relative_path"
-    FILE_NAME = "file_name"
-    LOCAL_FILE_PATH = "local_file_path"
-    SOURCE_FILE_URI = "source_file_relative_path"
-    FILE_FOLDER = "file_folder"
 
     def __init__(self) -> None:
         self._config = None
@@ -155,9 +148,9 @@ class AbstractFileBasedStreamReader(ABC):
         return False
 
     @abstractmethod
-    def upload(
+    def get_file(
         self, file: RemoteFile, local_directory: str, logger: logging.Logger
-    ) -> Tuple[FileRecordData, AirbyteRecordMessageFileReference]:
+    ) -> Dict[str, Any]:
         """
         This is required for connectors that will support writing to
         files. It will handle the logic to download,get,read,acquire or
@@ -169,41 +162,25 @@ class AbstractFileBasedStreamReader(ABC):
                logger (logging.Logger): Logger for logging information and errors.
 
            Returns:
-               AirbyteRecordMessageFileReference: A file reference object containing:
-                   - staging_file_url (str): The absolute path to the referenced file in the staging area.
-                   - file_size_bytes (int): The size of the referenced file in bytes.
-                   - source_file_relative_path (str): The relative path to the referenced file in source.
+               dict: A dictionary containing the following:
+                   - "file_url" (str): The absolute path of the downloaded file.
+                   - "bytes" (int): The file size in bytes.
+                   - "file_relative_path" (str): The relative path of the file for local storage. Is relative to local_directory as
+                   this a mounted volume in the pod container.
+
         """
         ...
 
-    def _get_file_transfer_paths(
-        self,
-        file: RemoteFile,
-        local_directory: str,
-        parse_file_path_from_uri: Optional[Callable[[str], str]] = None,
-    ) -> MutableMapping[str, Any]:
+    def _get_file_transfer_paths(self, file: RemoteFile, local_directory: str) -> List[str]:
         preserve_directory_structure = self.preserve_directory_structure()
-        if not parse_file_path_from_uri:
-            file_path = file.uri
-        else:
-            file_path = parse_file_path_from_uri(file.uri)
-
-        file_name = path.basename(file_path)
-        file_folder = path.dirname(file_path)
         if preserve_directory_structure:
             # Remove left slashes from source path format to make relative path for writing locally
-            file_relative_path = file_path.lstrip("/")
+            file_relative_path = file.uri.lstrip("/")
         else:
-            file_relative_path = file_name
+            file_relative_path = path.basename(file.uri)
         local_file_path = path.join(local_directory, file_relative_path)
+
         # Ensure the local directory exists
         makedirs(path.dirname(local_file_path), exist_ok=True)
-
-        file_paths = {
-            self.FILE_RELATIVE_PATH: file_relative_path,
-            self.LOCAL_FILE_PATH: local_file_path,
-            self.FILE_NAME: file_name,
-            self.FILE_FOLDER: file_folder,
-            self.SOURCE_FILE_URI: file.uri,
-        }
-        return file_paths
+        absolute_file_path = path.abspath(local_file_path)
+        return [file_relative_path, local_file_path, absolute_file_path]

airbyte_cdk/sources/file_based/file_types/file_transfer.py

@@ -2,27 +2,34 @@
 # Copyright (c) 2024 Airbyte, Inc., all rights reserved.
 #
 import logging
-from typing import Iterable, Tuple
+import os
+from typing import Any, Dict, Iterable
 
-from airbyte_cdk.models import AirbyteRecordMessageFileReference
+from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig
 from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader
-from airbyte_cdk.sources.file_based.file_record_data import FileRecordData
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
-from airbyte_cdk.sources.utils.files_directory import get_files_directory
+
+AIRBYTE_STAGING_DIRECTORY = os.getenv("AIRBYTE_STAGING_DIRECTORY", "/staging/files")
+DEFAULT_LOCAL_DIRECTORY = "/tmp/airbyte-file-transfer"
 
 
 class FileTransfer:
     def __init__(self) -> None:
-        self._local_directory = get_files_directory()
+        self._local_directory = (
+            AIRBYTE_STAGING_DIRECTORY
+            if os.path.exists(AIRBYTE_STAGING_DIRECTORY)
+            else DEFAULT_LOCAL_DIRECTORY
+        )
 
-    def upload(
+    def get_file(
         self,
+        config: FileBasedStreamConfig,
         file: RemoteFile,
         stream_reader: AbstractFileBasedStreamReader,
         logger: logging.Logger,
-    ) -> Iterable[Tuple[FileRecordData, AirbyteRecordMessageFileReference]]:
+    ) -> Iterable[Dict[str, Any]]:
         try:
-            yield stream_reader.upload(
+            yield stream_reader.get_file(
                 file=file, local_directory=self._local_directory, logger=logger
             )
         except Exception as ex:

airbyte_cdk/sources/file_based/schema_helpers.py

@@ -18,17 +18,9 @@ JsonSchemaSupportedType = Union[List[str], Literal["string"], str]
 SchemaType = Mapping[str, Mapping[str, JsonSchemaSupportedType]]
 
 schemaless_schema = {"type": "object", "properties": {"data": {"type": "object"}}}
-
 file_transfer_schema = {
     "type": "object",
-    "properties": {
-        "folder": {"type": "string"},
-        "file_name": {"type": "string"},
-        "bytes": {"type": "integer"},
-        "id": {"type": ["null", "string"]},
-        "updated_at": {"type": ["null", "string"]},
-        "mime_type": {"type": ["null", "string"]},
-    },
+    "properties": {"data": {"type": "object"}, "file": {"type": "object"}},
 }
 
 

airbyte_cdk/sources/file_based/stream/concurrent/adapters.py

@@ -4,7 +4,7 @@
 
 import copy
 import logging
-from functools import lru_cache
+from functools import cache, lru_cache
 from typing import TYPE_CHECKING, Any, Iterable, List, Mapping, MutableMapping, Optional, Union
 
 from typing_extensions import deprecated
@@ -258,14 +258,19 @@ class FileBasedStreamPartition(Partition):
                     and record_data.record is not None
                 ):
                     # `AirbyteMessage`s of type `Record` should also be yielded so they are enqueued
-                    record_message_data = record_data.record.data
+                    # If stream is flagged for file_transfer the record should data in file key
+                    record_message_data = (
+                        record_data.record.file
+                        if self._use_file_transfer()
+                        else record_data.record.data
+                    )
                     if not record_message_data:
                         raise ExceptionWithDisplayMessage("A record without data was found")
                     else:
                         yield Record(
                             data=record_message_data,
                             stream_name=self.stream_name(),
-                            file_reference=record_data.record.file_reference,
+                            is_file_transfer_message=self._use_file_transfer(),
                         )
                 else:
                     self._message_repository.emit_message(record_data)
@@ -301,6 +306,10 @@ class FileBasedStreamPartition(Partition):
     def stream_name(self) -> str:
         return self._stream.name
 
+    @cache
+    def _use_file_transfer(self) -> bool:
+        return hasattr(self._stream, "use_file_transfer") and self._stream.use_file_transfer
+
     def __repr__(self) -> str:
         return f"FileBasedStreamPartition({self._stream.name}, {self._slice})"
 

airbyte_cdk/sources/file_based/stream/default_file_based_stream.py

@@ -11,7 +11,7 @@ from functools import cache
 from os import path
 from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Set, Tuple, Union
 
-from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, AirbyteStream, FailureType, Level
+from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, FailureType, Level
 from airbyte_cdk.models import Type as MessageType
 from airbyte_cdk.sources.file_based.config.file_based_stream_config import PrimaryKeyType
 from airbyte_cdk.sources.file_based.exceptions import (
@@ -97,7 +97,14 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
         self, configured_catalog_json_schema: Dict[str, Any]
     ) -> Dict[str, Any]:
         if self.use_file_transfer:
-            return file_transfer_schema
+            return {
+                "type": "object",
+                "properties": {
+                    "file_path": {"type": "string"},
+                    "file_size": {"type": "string"},
+                    self.ab_file_name_col: {"type": "string"},
+                },
+            }
         else:
             return super()._filter_schema_invalid_properties(configured_catalog_json_schema)
 
@@ -138,6 +145,14 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
         record[self.ab_file_name_col] = file.uri
         return record
 
+    def transform_record_for_file_transfer(
+        self, record: dict[str, Any], file: RemoteFile
+    ) -> dict[str, Any]:
+        # timstamp() returns a float representing the number of seconds since the unix epoch
+        record[self.modified] = int(file.last_modified.timestamp()) * 1000
+        record[self.source_file_url] = file.uri
+        return record
+
     def read_records_from_slice(self, stream_slice: StreamSlice) -> Iterable[AirbyteMessage]:
         """
         Yield all records from all remote files in `list_files_for_this_sync`.
@@ -151,7 +166,6 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
             raise MissingSchemaError(FileBasedSourceError.MISSING_SCHEMA, stream=self.name)
         # The stream only supports a single file type, so we can use the same parser for all files
         parser = self.get_parser()
-        file_transfer = FileTransfer()
         for file in stream_slice["files"]:
             # only serialize the datetime once
             file_datetime_string = file.last_modified.strftime(self.DATE_TIME_FORMAT)
@@ -159,13 +173,19 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
 
             try:
                 if self.use_file_transfer:
-                    for file_record_data, file_reference in file_transfer.upload(
-                        file=file, stream_reader=self.stream_reader, logger=self.logger
+                    self.logger.info(f"{self.name}: {file} file-based syncing")
+                    # todo: complete here the code to not rely on local parser
+                    file_transfer = FileTransfer()
+                    for record in file_transfer.get_file(
+                        self.config, file, self.stream_reader, self.logger
                     ):
+                        line_no += 1
+                        if not self.record_passes_validation_policy(record):
+                            n_skipped += 1
+                            continue
+                        record = self.transform_record_for_file_transfer(record, file)
                         yield stream_data_to_airbyte_message(
-                            self.name,
-                            file_record_data.dict(exclude_none=True),
-                            file_reference=file_reference,
+                            self.name, record, is_file_transfer_message=True
                         )
                 else:
                     for record in parser.parse_records(
@@ -239,8 +259,6 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
 
     @cache
     def get_json_schema(self) -> JsonSchema:
-        if self.use_file_transfer:
-            return file_transfer_schema
         extra_fields = {
             self.ab_last_mod_col: {"type": "string"},
             self.ab_file_name_col: {"type": "string"},
@@ -264,7 +282,9 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
             return {"type": "object", "properties": {**extra_fields, **schema["properties"]}}
 
     def _get_raw_json_schema(self) -> JsonSchema:
-        if self.config.input_schema:
+        if self.use_file_transfer:
+            return file_transfer_schema
+        elif self.config.input_schema:
             return self.config.get_input_schema()  # type: ignore
         elif self.config.schemaless:
             return schemaless_schema
@@ -321,11 +341,6 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
             self.config.globs or [], self.config.legacy_prefix, self.logger
         )
 
-    def as_airbyte_stream(self) -> AirbyteStream:
-        file_stream = super().as_airbyte_stream()
-        file_stream.is_file_based = self.use_file_transfer
-        return file_stream
-
     def infer_schema(self, files: List[RemoteFile]) -> Mapping[str, Any]:
         loop = asyncio.get_event_loop()
         schema = loop.run_until_complete(self._infer_schema(files))

airbyte_cdk/sources/file_based/stream/permissions_file_based_stream.py

@@ -61,7 +61,9 @@ class PermissionsFileBasedStream(DefaultFileBasedStream):
                 permissions_record = self.transform_record(
                     permissions_record, file, file_datetime_string
                 )
-                yield stream_data_to_airbyte_message(self.name, permissions_record)
+                yield stream_data_to_airbyte_message(
+                    self.name, permissions_record, is_file_transfer_message=False
+                )
             except Exception as e:
                 self.logger.error(f"Failed to retrieve permissions for file {file.uri}: {str(e)}")
                 yield AirbyteMessage(

airbyte_cdk/sources/streams/concurrent/default_stream.py

@@ -29,7 +29,6 @@ class DefaultStream(AbstractStream):
         logger: Logger,
         cursor: Cursor,
         namespace: Optional[str] = None,
-        supports_file_transfer: bool = False,
     ) -> None:
         self._stream_partition_generator = partition_generator
         self._name = name
@@ -40,7 +39,6 @@ class DefaultStream(AbstractStream):
         self._logger = logger
         self._cursor = cursor
         self._namespace = namespace
-        self._supports_file_transfer = supports_file_transfer
 
     def generate_partitions(self) -> Iterable[Partition]:
         yield from self._stream_partition_generator.generate()
@@ -70,7 +68,6 @@ class DefaultStream(AbstractStream):
             json_schema=dict(self._json_schema),
             supported_sync_modes=[SyncMode.full_refresh],
             is_resumable=False,
-            is_file_based=self._supports_file_transfer,
         )
 
         if self._namespace:

airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py

@@ -71,6 +71,10 @@ class AbstractStreamStateConverter(ABC):
         for stream_slice in state.get("slices", []):
             stream_slice[self.START_KEY] = self._from_state_message(stream_slice[self.START_KEY])
             stream_slice[self.END_KEY] = self._from_state_message(stream_slice[self.END_KEY])
+            if self.MOST_RECENT_RECORD_KEY in stream_slice:
+                stream_slice[self.MOST_RECENT_RECORD_KEY] = self._from_state_message(
+                    stream_slice[self.MOST_RECENT_RECORD_KEY]
+                )
         return state
 
     def serialize(

airbyte_cdk/sources/types.py

@@ -6,7 +6,6 @@ from __future__ import annotations
 
 from typing import Any, ItemsView, Iterator, KeysView, List, Mapping, Optional, ValuesView
 
-from airbyte_cdk.models import AirbyteRecordMessageFileReference
 from airbyte_cdk.utils.slice_hasher import SliceHasher
 
 # A FieldPointer designates a path to a field inside a mapping. For example, retrieving ["k1", "k1.2"] in the object {"k1" :{"k1.2":
@@ -24,12 +23,12 @@ class Record(Mapping[str, Any]):
         data: Mapping[str, Any],
         stream_name: str,
         associated_slice: Optional[StreamSlice] = None,
-        file_reference: Optional[AirbyteRecordMessageFileReference] = None,
+        is_file_transfer_message: bool = False,
     ):
         self._data = data
         self._associated_slice = associated_slice
         self.stream_name = stream_name
-        self._file_reference = file_reference
+        self.is_file_transfer_message = is_file_transfer_message
 
     @property
     def data(self) -> Mapping[str, Any]:
@@ -39,14 +38,6 @@ class Record(Mapping[str, Any]):
     def associated_slice(self) -> Optional[StreamSlice]:
         return self._associated_slice
 
-    @property
-    def file_reference(self) -> AirbyteRecordMessageFileReference:
-        return self._file_reference
-
-    @file_reference.setter
-    def file_reference(self, value: AirbyteRecordMessageFileReference) -> None:
-        self._file_reference = value
-
     def __repr__(self) -> str:
         return repr(self._data)
 

airbyte_cdk/sources/utils/record_helper.py

@@ -9,10 +9,10 @@ from airbyte_cdk.models import (
     AirbyteLogMessage,
     AirbyteMessage,
     AirbyteRecordMessage,
-    AirbyteRecordMessageFileReference,
     AirbyteTraceMessage,
 )
 from airbyte_cdk.models import Type as MessageType
+from airbyte_cdk.models.file_transfer_record_message import AirbyteFileTransferRecordMessage
 from airbyte_cdk.sources.streams.core import StreamData
 from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer
 
@@ -22,7 +22,7 @@ def stream_data_to_airbyte_message(
     data_or_message: StreamData,
     transformer: TypeTransformer = TypeTransformer(TransformConfig.NoTransform),
     schema: Optional[Mapping[str, Any]] = None,
-    file_reference: Optional[AirbyteRecordMessageFileReference] = None,
+    is_file_transfer_message: bool = False,
 ) -> AirbyteMessage:
     if schema is None:
         schema = {}
@@ -36,12 +36,12 @@ def stream_data_to_airbyte_message(
             # taken unless configured. See
             # docs/connector-development/cdk-python/schemas.md for details.
             transformer.transform(data, schema)
-            message = AirbyteRecordMessage(
-                stream=stream_name,
-                data=data,
-                emitted_at=now_millis,
-                file_reference=file_reference,
-            )
+            if is_file_transfer_message:
+                message = AirbyteFileTransferRecordMessage(
+                    stream=stream_name, file=data, emitted_at=now_millis, data={}
+                )
+            else:
+                message = AirbyteRecordMessage(stream=stream_name, data=data, emitted_at=now_millis)
             return AirbyteMessage(type=MessageType.RECORD, record=message)
         case AirbyteTraceMessage():
             return AirbyteMessage(type=MessageType.TRACE, trace=data_or_message)

airbyte_cdk/test/mock_http/response_builder.py

@@ -198,14 +198,6 @@ def find_template(resource: str, execution_folder: str) -> Dict[str, Any]:
         return json.load(template_file)  # type: ignore  # we assume the dev correctly set up the resource file
 
 
-def find_binary_response(resource: str, execution_folder: str) -> bytes:
-    response_filepath = str(
-        get_unit_test_folder(execution_folder) / "resource" / "http" / "response" / f"{resource}"
-    )
-    with open(response_filepath, "rb") as response_file:
-        return response_file.read()  # type: ignore  # we assume the dev correctly set up the resource file
-
-
 def create_record_builder(
     response_template: Dict[str, Any],
     records_path: Union[FieldPath, NestedPath],

{airbyte_cdk-6.45.0.dev4107.dist-info → airbyte_cdk-6.45.0.post6.dev14369631849.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.45.0.dev4107
+Version: 6.45.0.post6.dev14369631849
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT
@@ -22,7 +22,7 @@ Provides-Extra: sql
 Provides-Extra: vector-db-based
 Requires-Dist: Jinja2 (>=3.1.2,<3.2.0)
 Requires-Dist: PyYAML (>=6.0.1,<7.0.0)
-Requires-Dist: airbyte-protocol-models-dataclasses (>=0.15,<0.16)
+Requires-Dist: airbyte-protocol-models-dataclasses (>=0.14,<0.15)
 Requires-Dist: anyascii (>=0.3.2,<0.4.0)
 Requires-Dist: avro (>=1.11.2,<1.13.0) ; extra == "file-based"
 Requires-Dist: backoff