airbyte-source-azure-blob-storage 0.6.15__tar.gz → 0.7.0__tar.gz

{airbyte_source_azure_blob_storage-0.6.15 → airbyte_source_azure_blob_storage-0.7.0}/PKG-INFO

@@ -1,18 +1,18 @@
 Metadata-Version: 2.1
 Name: airbyte-source-azure-blob-storage
-Version: 0.6.15
+Version: 0.7.0
 Summary: Source implementation for Azure Blob Storage.
 Home-page: https://airbyte.com
-License: MIT
+License: ELv2
 Author: Airbyte
 Author-email: contact@airbyte.io
 Requires-Python: >=3.11,<3.12
-Classifier: License :: OSI Approved :: MIT License
+Classifier: License :: Other/Proprietary License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: airbyte-cdk[file-based] (>=4,<5)
+Requires-Dist: airbyte-cdk[file-based] (>=7,<8)
 Requires-Dist: pytz (>=2024.1,<2025.0)
-Requires-Dist: smart-open[azure] (==0.6.15)
+Requires-Dist: smart-open[azure] (==0.7.0)
 Project-URL: Documentation, https://docs.airbyte.com/integrations/sources/azure-blob-storage
 Project-URL: Repository, https://github.com/airbytehq/airbyte
 Description-Content-Type: text/markdown

{airbyte_source_azure_blob_storage-0.6.15 → airbyte_source_azure_blob_storage-0.7.0}/pyproject.toml

@@ -3,11 +3,11 @@ requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "0.6.15"
+version = "0.7.0"
 name = "airbyte-source-azure-blob-storage"
 description = "Source implementation for Azure Blob Storage."
 authors = [ "Airbyte <contact@airbyte.io>",]
-license = "MIT"
+license = "ELv2"
 readme = "README.md"
 documentation = "https://docs.airbyte.com/integrations/sources/azure-blob-storage"
 homepage = "https://airbyte.com"
@@ -18,11 +18,11 @@ include = "source_azure_blob_storage"
 [tool.poetry.dependencies]
 python = "^3.11,<3.12"
 pytz = "^2024.1"
-airbyte-cdk = {extras = ["file-based"], version = "^4"}
+airbyte-cdk = {extras = ["file-based"], version = "^7"}
 
 [tool.poetry.dependencies.smart-open]
 extras = [ "azure",]
-version = "0.6.15"
+version = "0.7.0"
 
 [tool.poetry.scripts]
 source-azure-blob-storage = "source_azure_blob_storage.run:run"
@@ -32,7 +32,7 @@ docker = "^7.0.0"
 freezegun = "^1.4.0"
 pytest-mock = "^3.6.1"
 requests-mock = "^1.9.3"
-pandas = "2.2.1"
+pandas = "2.2.3"
 pytest = "^8.0.0"
 
 

{airbyte_source_azure_blob_storage-0.6.15 → airbyte_source_azure_blob_storage-0.7.0}/source_azure_blob_storage/config_migrations.py

@@ -7,7 +7,10 @@ import logging
 from abc import ABC, abstractmethod
 from typing import Any, List, Mapping
 
+import orjson
+
 from airbyte_cdk import AirbyteEntrypoint, Source, create_connector_config_control_message
+from airbyte_cdk.models import AirbyteMessageSerializer
 
 
 logger = logging.getLogger("airbyte_logger")
@@ -47,7 +50,7 @@ class MigrateConfig(ABC):
         Args:
         - migrated_config (Mapping[str, Any]): The migrated configuration.
         """
-        print(create_connector_config_control_message(migrated_config).json(exclude_unset=True))
+        print((orjson.dumps(AirbyteMessageSerializer.dump(create_connector_config_control_message(migrated_config))).decode()))
 
     @classmethod
     def migrate(cls, args: List[str], source: Source) -> None:

{airbyte_source_azure_blob_storage-0.6.15 → airbyte_source_azure_blob_storage-0.7.0}/source_azure_blob_storage/run.py

@@ -4,11 +4,13 @@
 
 
 import sys
+import time
 import traceback
-from datetime import datetime
+
+import orjson
 
 from airbyte_cdk import AirbyteEntrypoint, AirbyteMessage, Type, launch
-from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteTraceMessage, TraceType
+from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessageSerializer, AirbyteTraceMessage, TraceType
 from airbyte_cdk.sources.file_based.stream.cursor import DefaultFileBasedCursor
 from source_azure_blob_storage import SourceAzureBlobStorage, SourceAzureBlobStorageSpec, SourceAzureBlobStorageStreamReader
 from source_azure_blob_storage.config_migrations import MigrateCredentials, MigrateLegacyConfig
@@ -32,17 +34,21 @@ def run():
         MigrateCredentials.migrate(sys.argv[1:], source)
     except Exception:
         print(
-            AirbyteMessage(
-                type=Type.TRACE,
-                trace=AirbyteTraceMessage(
-                    type=TraceType.ERROR,
-                    emitted_at=int(datetime.now().timestamp() * 1000),
-                    error=AirbyteErrorTraceMessage(
-                        message="Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance.",
-                        stack_trace=traceback.format_exc(),
-                    ),
-                ),
-            ).json()
+            orjson.dumps(
+                AirbyteMessageSerializer.dump(
+                    AirbyteMessage(
+                        type=Type.TRACE,
+                        trace=AirbyteTraceMessage(
+                            type=TraceType.ERROR,
+                            emitted_at=time.time_ns() // 1_000_000,
+                            error=AirbyteErrorTraceMessage(
+                                message="Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance.",
+                                stack_trace=traceback.format_exc(),
+                            ),
+                        ),
+                    )
+                )
+            ).decode()
         )
     else:
         launch(source, args)

{airbyte_source_azure_blob_storage-0.6.15 → airbyte_source_azure_blob_storage-0.7.0}/source_azure_blob_storage/source.py

@@ -5,6 +5,8 @@
 
 from typing import Any
 
+from airbyte_protocol_dataclasses.models import AuthFlowType
+
 from airbyte_cdk import AdvancedAuth, ConnectorSpecification, OAuthConfigSpecification
 from airbyte_cdk.sources.file_based.file_based_source import FileBasedSource
 
@@ -19,7 +21,7 @@ class SourceAzureBlobStorage(FileBasedSource):
             documentationUrl=self.spec_class.documentation_url(),
             connectionSpecification=self.spec_class.schema(),
             advanced_auth=AdvancedAuth(
-                auth_flow_type="oauth2.0",
+                auth_flow_type=AuthFlowType.oauth2_0,
                 predicate_key=["credentials", "auth_type"],
                 predicate_value="oauth2",
                 oauth_config_specification=OAuthConfigSpecification(

{airbyte_source_azure_blob_storage-0.6.15 → airbyte_source_azure_blob_storage-0.7.0}/source_azure_blob_storage/spec.py

@@ -9,7 +9,7 @@ import dpath.util
 from pydantic.v1 import AnyUrl, BaseModel, Field
 
 from airbyte_cdk import OneOfOptionConfig
-from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec
+from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec, DeliverRawFiles, DeliverRecords
 
 
 class Oauth2(BaseModel):
@@ -107,6 +107,17 @@ class SourceAzureBlobStorageSpec(AbstractFileBasedSpec):
         order=11,
     )
 
+    delivery_method: Union[DeliverRecords, DeliverRawFiles] = Field(
+        title="Delivery Method",
+        discriminator="delivery_type",
+        type="object",
+        order=7,
+        display_type="radio",
+        group="advanced",
+        default="use_records_transfer",
+        airbyte_hidden=True,
+    )
+
     @classmethod
     def schema(cls, *args: Any, **kwargs: Any) -> Dict[str, Any]:
         """

{airbyte_source_azure_blob_storage-0.6.15 → airbyte_source_azure_blob_storage-0.7.0}/source_azure_blob_storage/stream_reader.py

@@ -9,11 +9,11 @@ import pytz
 from azure.core.credentials import AccessToken, TokenCredential
 from azure.core.exceptions import ResourceNotFoundError
 from azure.storage.blob import BlobServiceClient, ContainerClient
-from smart_open import open
+from smart_open import open as so_open
 
 from airbyte_cdk import AirbyteTracedException, FailureType
 from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode
-from airbyte_cdk.sources.file_based.remote_file import RemoteFile
+from airbyte_cdk.sources.file_based.remote_file import UploadableRemoteFile
 from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator
 
 from .spec import SourceAzureBlobStorageSpec
@@ -70,6 +70,26 @@ class AzureOauth2Authenticator(Oauth2Authenticator, TokenCredential):
         return AccessToken(token=self.get_access_token(), expires_on=7952342400)
 
 
+class AzureBlobStorageUploadableRemoteFile(UploadableRemoteFile):
+    blob_client: Any
+    blob_properties: Any
+
+    def __init__(self, blob_client: Any, blob_properties: Any, **kwargs):
+        super().__init__(**kwargs)
+        self.blob_client = blob_client
+        self.blob_properties = blob_properties
+
+    @property
+    def size(self) -> int:
+        return self.blob_properties.size
+
+    def download_to_local_directory(self, local_file_path: str) -> None:
+        blob_client = self.blob_client.get_blob_client(container=self.blob_properties.container, blob=self.uri)
+        with open(file=local_file_path, mode="wb") as f:
+            download_stream = blob_client.download_blob()
+            f.write(download_stream.readall())
+
+
 class SourceAzureBlobStorageStreamReader(AbstractFileBasedStreamReader):
     _credentials = None
 
@@ -127,20 +147,29 @@ class SourceAzureBlobStorageStreamReader(AbstractFileBasedStreamReader):
         globs: List[str],
         prefix: Optional[str],
         logger: logging.Logger,
-    ) -> Iterable[RemoteFile]:
+    ) -> Iterable[AzureBlobStorageUploadableRemoteFile]:
         prefixes = [prefix] if prefix else self.get_prefixes_from_globs(globs)
         prefixes = prefixes or [None]
         try:
             for prefix in prefixes:
                 for blob in self.azure_container_client.list_blobs(name_starts_with=prefix):
-                    remote_file = RemoteFile(uri=blob.name, last_modified=blob.last_modified.astimezone(pytz.utc).replace(tzinfo=None))
+                    remote_file = AzureBlobStorageUploadableRemoteFile(
+                        uri=blob.name,
+                        last_modified=blob.last_modified.astimezone(pytz.utc).replace(tzinfo=None),
+                        blob_client=self.azure_blob_service_client,
+                        blob_properties=blob,
+                        created_at=blob.creation_time.astimezone(pytz.utc).replace(tzinfo=None).strftime("%Y-%m-%dT%H:%M:%S.%fZ"),
+                        updated_at=blob.last_modified.astimezone(pytz.utc).replace(tzinfo=None).strftime("%Y-%m-%dT%H:%M:%S.%fZ"),
+                    )
                     yield from self.filter_files_by_globs_and_start_date([remote_file], globs)
         except ResourceNotFoundError as e:
             raise AirbyteTracedException(failure_type=FailureType.config_error, internal_message=e.message, message=e.reason or e.message)
 
-    def open_file(self, file: RemoteFile, mode: FileReadMode, encoding: Optional[str], logger: logging.Logger) -> IOBase:
+    def open_file(
+        self, file: AzureBlobStorageUploadableRemoteFile, mode: FileReadMode, encoding: Optional[str], logger: logging.Logger
+    ) -> IOBase:
         try:
-            result = open(
+            result = so_open(
                 f"azure://{self.config.azure_blob_storage_container_name}/{file.uri}",
                 transport_params={"client": self.azure_blob_service_client},
                 mode=mode.value,