airbyte-source-s3 4.14.0.dev202504091813__tar.gz → 4.15.1__tar.gz

{airbyte_source_s3-4.14.0.dev202504091813 → airbyte_source_s3-4.15.1}/PKG-INFO

@@ -1,25 +1,27 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.1
 Name: airbyte-source-s3
-Version: 4.14.0.dev202504091813
+Version: 4.15.1
 Summary: Source implementation for S3.
+Home-page: https://airbyte.com
 License: ELv2
 Author: Airbyte
 Author-email: contact@airbyte.io
-Requires-Python: >=3.10,<3.12
+Requires-Python: >=3.10,<3.14
 Classifier: License :: Other/Proprietary License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: airbyte-cdk[file-based] (==6.45.0.dev04107)
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: airbyte-cdk[file-based] (>=7.0.4,<8.0.0)
 Requires-Dist: dill (>=0.3.4,<0.4.0)
 Requires-Dist: pendulum (>=3.0.0,<4.0.0)
 Requires-Dist: pytz (>=2024.2,<2025.0)
-Requires-Dist: smart-open[s3] (>=5.1.0,<6.0.0)
+Requires-Dist: smart-open[s3] (==4.15.1)
 Requires-Dist: transformers (>=4.38.2,<5.0.0)
 Requires-Dist: urllib3 (<2)
 Requires-Dist: wcmatch (>=10.0,<11.0)
 Project-URL: Documentation, https://docs.airbyte.com/integrations/sources/s3
-Project-URL: Homepage, https://airbyte.com
 Project-URL: Repository, https://github.com/airbytehq/airbyte
 Description-Content-Type: text/markdown
 

{airbyte_source_s3-4.14.0.dev202504091813 → airbyte_source_s3-4.15.1}/pyproject.toml

@@ -1,49 +1,37 @@
 [build-system]
-requires = [
-    "poetry-core>=1.0.0",
-]
+requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "4.14.0.dev202504091813"
+version = "4.15.1"
 name = "airbyte-source-s3"
 description = "Source implementation for S3."
-authors = [
-    "Airbyte <contact@airbyte.io>",
-]
+authors = [ "Airbyte <contact@airbyte.io>",]
 license = "ELv2"
 readme = "README.md"
 documentation = "https://docs.airbyte.com/integrations/sources/s3"
 homepage = "https://airbyte.com"
 repository = "https://github.com/airbytehq/airbyte"
-packages = [
-    { include = "source_s3" },
-]
+[[tool.poetry.packages]]
+include = "source_s3"
 
 [tool.poetry.dependencies]
-python = "^3.10,<3.12"
+python = "^3.10,<3.14"
 pytz = "^2024.2"
 wcmatch = "^10.0"
 dill = "^0.3.4"
 transformers = "^4.38.2"
 urllib3 = "<2"
+airbyte-cdk = {extras = ["file-based"], version = "^7.0.4"}
 pendulum = "^3.0.0"
 
-[tool.poetry.dependencies.airbyte-cdk]
-extras = [
-    "file-based",
-]
-version = "6.45.0.dev04107"
-
-[tool.poetry.dependencies.smart-open]
-extras = [
-    "s3",
-]
-version = "^5.1.0"
-
 [tool.poetry.scripts]
 source-s3 = "source_s3.run:run"
 
+[tool.poetry.dependencies.smart-open]
+extras = [ "s3",]
+version = "4.15.1"
+
 [tool.poetry.group.dev.dependencies]
 pytest = "^8.0.0"
 moto = "==4.2.14"
@@ -54,5 +42,7 @@ pandas = "^2.0.3"
 
 [tool.poe]
 include = [
+    # Shared tasks definition file(s) can be imported here.
+    # Run `poe` or `poe --help` to see the list of available tasks.
     "${POE_GIT_DIR}/poe-tasks/poetry-connector-tasks.toml",
 ]

{airbyte_source_s3-4.14.0.dev202504091813 → airbyte_source_s3-4.15.1}/source_s3/v4/stream_reader.py

@@ -168,6 +168,19 @@ class SourceS3StreamReader(AbstractFileBasedStreamReader):
             endpoint=self.config.endpoint,
         ) from exc
 
+    def _construct_s3_uri(self, file: RemoteFile) -> str:
+        """
+        Constructs the S3 URI for a given file, handling both regular files and files inside archives.
+
+        Args:
+            file: The RemoteFile object representing either a regular file or a file inside an archive
+
+        Returns:
+            str: The properly formatted S3 URI
+        """
+        file_path = file.uri.split("#")[0] if isinstance(file, RemoteFileInsideArchive) else file.uri
+        return f"s3://{self.config.bucket}/{file_path}"
+
     def open_file(self, file: RemoteFile, mode: FileReadMode, encoding: Optional[str], logger: logging.Logger) -> IOBase:
         try:
             params = {"client": self.s3_client}
@@ -176,14 +189,13 @@ class SourceS3StreamReader(AbstractFileBasedStreamReader):
 
         logger.debug(f"try to open {file.uri}")
         try:
+            s3_uri = self._construct_s3_uri(file)
             if isinstance(file, RemoteFileInsideArchive):
-                s3_file_object = smart_open.open(f"s3://{self.config.bucket}/{file.uri.split('#')[0]}", transport_params=params, mode="rb")
+                s3_file_object = smart_open.open(s3_uri, transport_params=params, mode="rb")
                 decompressed_stream = DecompressedStream(s3_file_object, file)
                 result = ZipContentReader(decompressed_stream, encoding)
             else:
-                result = smart_open.open(
-                    f"s3://{self.config.bucket}/{file.uri}", transport_params=params, mode=mode.value, encoding=encoding
-                )
+                result = smart_open.open(s3_uri, transport_params=params, mode=mode.value, encoding=encoding)
         except OSError:
             logger.warning(
                 f"We don't have access to {file.uri}. The file appears to have become unreachable during sync."
@@ -247,7 +259,7 @@ class SourceS3StreamReader(AbstractFileBasedStreamReader):
             message = "File size exceeds the 1 GB limit."
             raise FileSizeLimitError(message=message, internal_message=message, failure_type=FailureType.config_error)
 
-        file_paths = self._get_file_transfer_paths(file, local_directory)
+        file_paths = self._get_file_transfer_paths(file.uri, local_directory)
         local_file_path = file_paths[self.LOCAL_FILE_PATH]
         file_relative_path = file_paths[self.FILE_RELATIVE_PATH]
         file_name = file_paths[self.FILE_NAME]
@@ -264,9 +276,10 @@ class SourceS3StreamReader(AbstractFileBasedStreamReader):
 
         file_record_data = FileRecordData(
             folder=file_paths[self.FILE_FOLDER],
-            filename=file_name,
+            file_name=file_name,
             bytes=file_size,
             updated_at=file.last_modified.strftime("%Y-%m-%dT%H:%M:%S.%fZ"),
+            source_uri=self._construct_s3_uri(file),
         )
 
         file_reference = AirbyteRecordMessageFileReference(

{airbyte_source_s3-4.14.0.dev202504091813 → airbyte_source_s3-4.15.1}/source_s3/v4/zip_reader.py

@@ -366,7 +366,21 @@ class ZipContentReader:
         data = self.buffer[:size]
         self.buffer = self.buffer[size:]
 
-        return data.decode(self.encoding) if self.encoding else bytes(data)
+        try:
+            return data.decode(self.encoding) if self.encoding else bytes(data)
+        except UnicodeDecodeError:
+            if self.encoding == "utf_8_sig":
+                # utf_8_sig considers `\xef\xbb\xbf` as a single character and therefore calling `bytearray(b'\xef').decode("utf_8_sig") will
+                # cause an exception to be raised.
+                number_of_bytes_to_add = size - 1
+                if data.endswith(bytearray(b"\xef")):
+                    number_of_bytes_to_add += 2
+                elif data.endswith(bytearray(b"\xbb")):
+                    number_of_bytes_to_add += 1
+                data = data + self.buffer[:number_of_bytes_to_add]
+                self.buffer = self.buffer[number_of_bytes_to_add:]
+                return data.decode(self.encoding) if self.encoding else bytes(data)
+            raise
 
     def seek(self, offset: int, whence: int = io.SEEK_SET) -> int:
         """