airbyte-source-azure-blob-storage 0.6.9__py3-none-any.whl → 0.8.2__py3-none-any.whl

{airbyte_source_azure_blob_storage-0.6.9.dist-info → airbyte_source_azure_blob_storage-0.8.2.dist-info}/METADATA

@@ -1,19 +1,21 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.1
 Name: airbyte-source-azure-blob-storage
-Version: 0.6.9
+Version: 0.8.2
 Summary: Source implementation for Azure Blob Storage.
-License: MIT
+Home-page: https://airbyte.com
+License: ELv2
 Author: Airbyte
 Author-email: contact@airbyte.io
-Requires-Python: >=3.11,<3.12
-Classifier: License :: OSI Approved :: MIT License
+Requires-Python: >=3.11,<3.14
+Classifier: License :: Other/Proprietary License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: airbyte-cdk[file-based] (>=4,<5)
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: airbyte-cdk[file-based] (>=7.0.0,<8.0.0)
 Requires-Dist: pytz (>=2024.1,<2025.0)
-Requires-Dist: smart-open[azure] (==6.4.0)
+Requires-Dist: smart-open[azure] (==0.8.2)
 Project-URL: Documentation, https://docs.airbyte.com/integrations/sources/azure-blob-storage
-Project-URL: Homepage, https://airbyte.com
 Project-URL: Repository, https://github.com/airbytehq/airbyte
 Description-Content-Type: text/markdown
 

airbyte_source_azure_blob_storage-0.8.2.dist-info/RECORD

@@ -0,0 +1,10 @@
+source_azure_blob_storage/__init__.py,sha256=Wx4PzvHg900-c2CpOOP1Wk0zcJpNVqJrkMnPtDcuaQM,319
+source_azure_blob_storage/config_migrations.py,sha256=b0_UyUj1F4q_wPCxBpu4W8MxfMiHE2OvesXxks1bL2M,4331
+source_azure_blob_storage/run.py,sha256=4FImO8txeARJAE-c-glw5xxnDuiRT-XSLYKFVBn70tI,2206
+source_azure_blob_storage/source.py,sha256=ZubuoAmy_QxRtJIJVSugmBoMxw3dMjLTSAIXbZf3jjE,2493
+source_azure_blob_storage/spec.py,sha256=tRiF9KacJFK97HaleJucQadR4e2hjnmStJnbLEFV4vc,5043
+source_azure_blob_storage/stream_reader.py,sha256=YWzmk1mPqSy8H8gWRBYZc7F7-vBDqlNH3WrhVFCmAmk,7978
+airbyte_source_azure_blob_storage-0.8.2.dist-info/METADATA,sha256=SZFa65iIEWD4KAqg_sv4VI9ptsDz4-Xc7x9jlyle4zs,6282
+airbyte_source_azure_blob_storage-0.8.2.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+airbyte_source_azure_blob_storage-0.8.2.dist-info/entry_points.txt,sha256=75v_DA_Xu0qr0eqtEXyh8sPCqcL9eXKWY8UwdST3ANE,79
+airbyte_source_azure_blob_storage-0.8.2.dist-info/RECORD,,

{airbyte_source_azure_blob_storage-0.6.9.dist-info → airbyte_source_azure_blob_storage-0.8.2.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 2.1.3
+Generator: poetry-core 1.9.1
 Root-Is-Purelib: true
 Tag: py3-none-any

source_azure_blob_storage/config_migrations.py

@@ -7,7 +7,10 @@ import logging
 from abc import ABC, abstractmethod
 from typing import Any, List, Mapping
 
+import orjson
+
 from airbyte_cdk import AirbyteEntrypoint, Source, create_connector_config_control_message
+from airbyte_cdk.models import AirbyteMessageSerializer
 
 
 logger = logging.getLogger("airbyte_logger")
@@ -47,7 +50,7 @@ class MigrateConfig(ABC):
         Args:
         - migrated_config (Mapping[str, Any]): The migrated configuration.
         """
-        print(create_connector_config_control_message(migrated_config).json(exclude_unset=True))
+        print((orjson.dumps(AirbyteMessageSerializer.dump(create_connector_config_control_message(migrated_config))).decode()))
 
     @classmethod
     def migrate(cls, args: List[str], source: Source) -> None:

source_azure_blob_storage/run.py

@@ -4,11 +4,13 @@
 
 
 import sys
+import time
 import traceback
-from datetime import datetime
+
+import orjson
 
 from airbyte_cdk import AirbyteEntrypoint, AirbyteMessage, Type, launch
-from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteTraceMessage, TraceType
+from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessageSerializer, AirbyteTraceMessage, TraceType
 from airbyte_cdk.sources.file_based.stream.cursor import DefaultFileBasedCursor
 from source_azure_blob_storage import SourceAzureBlobStorage, SourceAzureBlobStorageSpec, SourceAzureBlobStorageStreamReader
 from source_azure_blob_storage.config_migrations import MigrateCredentials, MigrateLegacyConfig
@@ -32,17 +34,21 @@ def run():
         MigrateCredentials.migrate(sys.argv[1:], source)
     except Exception:
         print(
-            AirbyteMessage(
-                type=Type.TRACE,
-                trace=AirbyteTraceMessage(
-                    type=TraceType.ERROR,
-                    emitted_at=int(datetime.now().timestamp() * 1000),
-                    error=AirbyteErrorTraceMessage(
-                        message="Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance.",
-                        stack_trace=traceback.format_exc(),
-                    ),
-                ),
-            ).json()
+            orjson.dumps(
+                AirbyteMessageSerializer.dump(
+                    AirbyteMessage(
+                        type=Type.TRACE,
+                        trace=AirbyteTraceMessage(
+                            type=TraceType.ERROR,
+                            emitted_at=time.time_ns() // 1_000_000,
+                            error=AirbyteErrorTraceMessage(
+                                message="Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance.",
+                                stack_trace=traceback.format_exc(),
+                            ),
+                        ),
+                    )
+                )
+            ).decode()
         )
     else:
         launch(source, args)

source_azure_blob_storage/source.py

@@ -5,6 +5,8 @@
 
 from typing import Any
 
+from airbyte_protocol_dataclasses.models import AuthFlowType
+
 from airbyte_cdk import AdvancedAuth, ConnectorSpecification, OAuthConfigSpecification
 from airbyte_cdk.sources.file_based.file_based_source import FileBasedSource
 
@@ -19,7 +21,7 @@ class SourceAzureBlobStorage(FileBasedSource):
             documentationUrl=self.spec_class.documentation_url(),
             connectionSpecification=self.spec_class.schema(),
             advanced_auth=AdvancedAuth(
-                auth_flow_type="oauth2.0",
+                auth_flow_type=AuthFlowType.oauth2_0,
                 predicate_key=["credentials", "auth_type"],
                 predicate_value="oauth2",
                 oauth_config_specification=OAuthConfigSpecification(

source_azure_blob_storage/spec.py

@@ -9,7 +9,7 @@ import dpath.util
 from pydantic.v1 import AnyUrl, BaseModel, Field
 
 from airbyte_cdk import OneOfOptionConfig
-from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec
+from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec, DeliverRawFiles, DeliverRecords
 
 
 class Oauth2(BaseModel):
@@ -107,6 +107,17 @@ class SourceAzureBlobStorageSpec(AbstractFileBasedSpec):
         order=11,
     )
 
+    delivery_method: Union[DeliverRecords, DeliverRawFiles] = Field(
+        title="Delivery Method",
+        discriminator="delivery_type",
+        type="object",
+        order=7,
+        display_type="radio",
+        group="advanced",
+        default="use_records_transfer",
+        airbyte_hidden=True,
+    )
+
     @classmethod
     def schema(cls, *args: Any, **kwargs: Any) -> Dict[str, Any]:
         """

source_azure_blob_storage/stream_reader.py

@@ -9,11 +9,11 @@ import pytz
 from azure.core.credentials import AccessToken, TokenCredential
 from azure.core.exceptions import ResourceNotFoundError
 from azure.storage.blob import BlobServiceClient, ContainerClient
-from smart_open import open
+from smart_open import open as so_open
 
 from airbyte_cdk import AirbyteTracedException, FailureType
 from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode
-from airbyte_cdk.sources.file_based.remote_file import RemoteFile
+from airbyte_cdk.sources.file_based.remote_file import UploadableRemoteFile
 from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator
 
 from .spec import SourceAzureBlobStorageSpec
@@ -70,6 +70,26 @@ class AzureOauth2Authenticator(Oauth2Authenticator, TokenCredential):
         return AccessToken(token=self.get_access_token(), expires_on=7952342400)
 
 
+class AzureBlobStorageUploadableRemoteFile(UploadableRemoteFile):
+    blob_client: Any
+    blob_properties: Any
+
+    def __init__(self, blob_client: Any, blob_properties: Any, **kwargs):
+        super().__init__(**kwargs)
+        self.blob_client = blob_client
+        self.blob_properties = blob_properties
+
+    @property
+    def size(self) -> int:
+        return self.blob_properties.size
+
+    def download_to_local_directory(self, local_file_path: str) -> None:
+        blob_client = self.blob_client.get_blob_client(container=self.blob_properties.container, blob=self.uri)
+        with open(file=local_file_path, mode="wb") as f:
+            download_stream = blob_client.download_blob()
+            f.write(download_stream.readall())
+
+
 class SourceAzureBlobStorageStreamReader(AbstractFileBasedStreamReader):
     _credentials = None
 
@@ -127,20 +147,29 @@ class SourceAzureBlobStorageStreamReader(AbstractFileBasedStreamReader):
         globs: List[str],
         prefix: Optional[str],
         logger: logging.Logger,
-    ) -> Iterable[RemoteFile]:
+    ) -> Iterable[AzureBlobStorageUploadableRemoteFile]:
         prefixes = [prefix] if prefix else self.get_prefixes_from_globs(globs)
         prefixes = prefixes or [None]
         try:
             for prefix in prefixes:
                 for blob in self.azure_container_client.list_blobs(name_starts_with=prefix):
-                    remote_file = RemoteFile(uri=blob.name, last_modified=blob.last_modified.astimezone(pytz.utc).replace(tzinfo=None))
+                    remote_file = AzureBlobStorageUploadableRemoteFile(
+                        uri=blob.name,
+                        last_modified=blob.last_modified.astimezone(pytz.utc).replace(tzinfo=None),
+                        blob_client=self.azure_blob_service_client,
+                        blob_properties=blob,
+                        created_at=blob.creation_time.astimezone(pytz.utc).replace(tzinfo=None).strftime("%Y-%m-%dT%H:%M:%S.%fZ"),
+                        updated_at=blob.last_modified.astimezone(pytz.utc).replace(tzinfo=None).strftime("%Y-%m-%dT%H:%M:%S.%fZ"),
+                    )
                     yield from self.filter_files_by_globs_and_start_date([remote_file], globs)
         except ResourceNotFoundError as e:
             raise AirbyteTracedException(failure_type=FailureType.config_error, internal_message=e.message, message=e.reason or e.message)
 
-    def open_file(self, file: RemoteFile, mode: FileReadMode, encoding: Optional[str], logger: logging.Logger) -> IOBase:
+    def open_file(
+        self, file: AzureBlobStorageUploadableRemoteFile, mode: FileReadMode, encoding: Optional[str], logger: logging.Logger
+    ) -> IOBase:
         try:
-            result = open(
+            result = so_open(
                 f"azure://{self.config.azure_blob_storage_container_name}/{file.uri}",
                 transport_params={"client": self.azure_blob_service_client},
                 mode=mode.value,

airbyte_source_azure_blob_storage-0.6.9.dist-info/RECORD

@@ -1,10 +0,0 @@
-source_azure_blob_storage/__init__.py,sha256=Wx4PzvHg900-c2CpOOP1Wk0zcJpNVqJrkMnPtDcuaQM,319
-source_azure_blob_storage/config_migrations.py,sha256=WKo5ETt4y09_cjfny3EH4filBsC48ltLMlQ_wLcxYVM,4229
-source_azure_blob_storage/run.py,sha256=3rwgY4Qs-KdOKZanpiLmGsQzNW5gUEE9zmrizEpCs20,1999
-source_azure_blob_storage/source.py,sha256=j5ibzjzX_gAi2kASpajoQ8ZMbnJ8LIwuSKChwk5i3vY,2420
-source_azure_blob_storage/spec.py,sha256=ysnDed3VwUmQnBhOYHHkWivUH3Q7EqDbjvT1Qqc7fPo,4697
-source_azure_blob_storage/stream_reader.py,sha256=ZSMgiwy6dOsFAGPyDIwCS4SklNIRo7dKdwesWIrsCrQ,6674
-airbyte_source_azure_blob_storage-0.6.9.dist-info/METADATA,sha256=z6Ag3OgwXS9oZ4vRrR5DmRocBY0wNV-KdFYW7SAhmeE,6185
-airbyte_source_azure_blob_storage-0.6.9.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-airbyte_source_azure_blob_storage-0.6.9.dist-info/entry_points.txt,sha256=75v_DA_Xu0qr0eqtEXyh8sPCqcL9eXKWY8UwdST3ANE,79
-airbyte_source_azure_blob_storage-0.6.9.dist-info/RECORD,,