airbyte-cdk 6.45.0.dev4100__py3-none-any.whl → 6.45.0.dev4102__py3-none-any.whl

airbyte_cdk/sources/file_based/file_based_stream_reader.py

@@ -8,7 +8,7 @@ from datetime import datetime
 from enum import Enum
 from io import IOBase
 from os import makedirs, path
-from typing import Iterable, List, Optional, Set, Tuple
+from typing import Any, Iterable, List, Optional, Set, Tuple, MutableMapping
 
 from wcmatch.glob import GLOBSTAR, globmatch
 
@@ -30,6 +30,12 @@ class FileReadMode(Enum):
 
 class AbstractFileBasedStreamReader(ABC):
     DATE_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
+    FILE_RELATIVE_PATH = "file_relative_path"
+    FILE_NAME = "file_name"
+    LOCAL_FILE_PATH = "local_file_path"
+    ABSOLUTE_FILE_PATH = "absolute_file_path"
+    SOURCE_FILE_URI = "source_file_relative_path"
+    FILE_FOLDER = "file_folder"
 
     def __init__(self) -> None:
         self._config = None
@@ -171,16 +177,28 @@ class AbstractFileBasedStreamReader(ABC):
         """
         ...
 
-    def _get_file_transfer_paths(self, file: RemoteFile, local_directory: str) -> List[str]:
+    def _get_file_transfer_paths(self, file: RemoteFile, local_directory: str) -> MutableMapping[str, Any]:
         preserve_directory_structure = self.preserve_directory_structure()
+        file_uri = file.uri
+        file_name = path.basename(file_uri)
+        file_folder = path.dirname(file_uri)
         if preserve_directory_structure:
             # Remove left slashes from source path format to make relative path for writing locally
-            file_relative_path = file.uri.lstrip("/")
+            file_relative_path = file_uri.lstrip("/")
         else:
-            file_relative_path = path.basename(file.uri)
+            file_relative_path = file_name
         local_file_path = path.join(local_directory, file_relative_path)
-
         # Ensure the local directory exists
         makedirs(path.dirname(local_file_path), exist_ok=True)
         absolute_file_path = path.abspath(local_file_path)
-        return [file_relative_path, local_file_path, absolute_file_path]
+
+        file_paths = {
+            self.FILE_RELATIVE_PATH: file_relative_path,
+            self.LOCAL_FILE_PATH: local_file_path,
+            self.ABSOLUTE_FILE_PATH: absolute_file_path,
+            self.FILE_NAME: file_name,
+            self.FILE_FOLDER: file_folder,
+            self.SOURCE_FILE_URI: file_uri,
+
+        }
+        return file_paths

airbyte_cdk/sources/file_based/stream/concurrent/adapters.py

@@ -4,7 +4,7 @@
 
 import copy
 import logging
-from functools import cache, lru_cache
+from functools import lru_cache
 from typing import TYPE_CHECKING, Any, Iterable, List, Mapping, MutableMapping, Optional, Union
 
 from typing_extensions import deprecated
@@ -258,12 +258,7 @@ class FileBasedStreamPartition(Partition):
                     and record_data.record is not None
                 ):
                     # `AirbyteMessage`s of type `Record` should also be yielded so they are enqueued
-                    # If stream is flagged for file_transfer the record should data in file key
-                    record_message_data = (
-                        record_data.record.file
-                        if self._use_file_transfer()
-                        else record_data.record.data
-                    )
+                    record_message_data = record_data.record.data
                     if not record_message_data:
                         raise ExceptionWithDisplayMessage("A record without data was found")
                     else:
@@ -305,10 +300,6 @@ class FileBasedStreamPartition(Partition):
     def stream_name(self) -> str:
         return self._stream.name
 
-    @cache
-    def _use_file_transfer(self) -> bool:
-        return hasattr(self._stream, "use_file_transfer") and self._stream.use_file_transfer
-
     def __repr__(self) -> str:
         return f"FileBasedStreamPartition({self._stream.name}, {self._slice})"
 

{airbyte_cdk-6.45.0.dev4100.dist-info → airbyte_cdk-6.45.0.dev4102.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.45.0.dev4100
+Version: 6.45.0.dev4102
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT

{airbyte_cdk-6.45.0.dev4100.dist-info → airbyte_cdk-6.45.0.dev4102.dist-info}/RECORD

@@ -216,7 +216,7 @@ airbyte_cdk/sources/file_based/discovery_policy/default_discovery_policy.py,sha2
 airbyte_cdk/sources/file_based/exceptions.py,sha256=WP0qkG6fpWoBpOyyicgp5YNE393VWyegq5qSy0v4QtM,7362
 airbyte_cdk/sources/file_based/file_based_source.py,sha256=bIsut7ivHcl7YPO9cygDn0to23MRLu1Sym7jSr0Iy9A,20051
 airbyte_cdk/sources/file_based/file_based_stream_permissions_reader.py,sha256=4e7FXqQ9hueacexC0SyrZyjF8oREYHza8pKF9CgKbD8,5050
-airbyte_cdk/sources/file_based/file_based_stream_reader.py,sha256=K19ejtmWLWt-q6ANXGy_VW6eMzVvkbkIaBVoOK3ahA8,6906
+airbyte_cdk/sources/file_based/file_based_stream_reader.py,sha256=r6pjkBfCigTKFd_Z5fGpGR1Ovwxf9rzoFRlec79PGIw,7565
 airbyte_cdk/sources/file_based/file_record_data.py,sha256=HXduy_owayUhgmbuswOl5UnszWPUAkRiNhDJgZVCVZI,471
 airbyte_cdk/sources/file_based/file_types/__init__.py,sha256=blCLn0-2LC-ZdgcNyDEhqM2RiUvEjEBh-G4-t32ZtuM,1268
 airbyte_cdk/sources/file_based/file_types/avro_parser.py,sha256=XNx-JC-sgzH9u3nOJ2M59FxBXvtig8LN6BIkeDOavZA,10858
@@ -235,7 +235,7 @@ airbyte_cdk/sources/file_based/schema_validation_policies/default_schema_validat
 airbyte_cdk/sources/file_based/stream/__init__.py,sha256=q_zmeOHHg0JK5j1YNSOIsyXGz-wlTl_0E8z5GKVAcVM,543
 airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py,sha256=9pQh3BHYcxm8CRC8XawfmBxL8O9HggpWwCCbX_ncINE,7509
 airbyte_cdk/sources/file_based/stream/concurrent/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-airbyte_cdk/sources/file_based/stream/concurrent/adapters.py,sha256=2a9-wSbES-Ehh17WR0WHRuEhUAdhRm7gSl0Ztixzrwo,13851
+airbyte_cdk/sources/file_based/stream/concurrent/adapters.py,sha256=whExivz7qYc561SRy-MJHk7pKt9wK-21Q0GRKd8u1FY,13447
 airbyte_cdk/sources/file_based/stream/concurrent/cursor/__init__.py,sha256=Rx7TwjH8B7e0eee83Tlqxv1bWn-BVXOmlUAH7auM1uM,344
 airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py,sha256=5dYZMLBEbvCyrCT89lCYdm2FdrLPLuxjdpQSVGP5o0w,1856
 airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py,sha256=gRTL-9I3ejjQOpLKd6ixe9rB3kGlubCdhUt9ri6AdAI,14880
@@ -360,9 +360,9 @@ airbyte_cdk/utils/slice_hasher.py,sha256=EDxgROHDbfG-QKQb59m7h_7crN1tRiawdf5uU7G
 airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
 airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
 airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
-airbyte_cdk-6.45.0.dev4100.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
-airbyte_cdk-6.45.0.dev4100.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
-airbyte_cdk-6.45.0.dev4100.dist-info/METADATA,sha256=76QoD2QmXEsG_pl2jSBzdCfwXFPPfSHMVZjXs98qkSg,6089
-airbyte_cdk-6.45.0.dev4100.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-airbyte_cdk-6.45.0.dev4100.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
-airbyte_cdk-6.45.0.dev4100.dist-info/RECORD,,
+airbyte_cdk-6.45.0.dev4102.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
+airbyte_cdk-6.45.0.dev4102.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
+airbyte_cdk-6.45.0.dev4102.dist-info/METADATA,sha256=jAf-_v4EfZ32YxJ8JhbFIwyOf2wPoEO00-srzm-X4nA,6089
+airbyte_cdk-6.45.0.dev4102.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+airbyte_cdk-6.45.0.dev4102.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
+airbyte_cdk-6.45.0.dev4102.dist-info/RECORD,,