airbyte-source-azure-blob-storage 0.3.5__py3-none-any.whl → 0.4.0__py3-none-any.whl

{airbyte_source_azure_blob_storage-0.3.5.dist-info → airbyte_source_azure_blob_storage-0.4.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-source-azure-blob-storage
-Version: 0.3.5
+Version: 0.4.0
 Summary: Source implementation for Azure Blob Storage.
 Home-page: https://airbyte.com
 License: MIT
@@ -12,7 +12,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: airbyte-cdk[file-based] (==0.75)
+Requires-Dist: airbyte-cdk[file-based] (>=0,<1)
 Requires-Dist: pytz (==2024.1)
 Requires-Dist: smart-open[azure] (==6.4.0)
 Project-URL: Documentation, https://docs.airbyte.com/integrations/sources/azure-blob-storage
@@ -32,6 +32,26 @@ For information about how to use this connector within Airbyte, see [the documen
 * Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation)
 
 
+### Generate new oauth token
+
+Tenant id should be provided by user, reason:
+https://learn.microsoft.com/en-us/answers/questions/1531138/which-tenant-id-do-i-have-to-use-to-get-tokens-and
+
+1. GET https://login.microsoftonline.com/<tenant_id>/oauth2/v2.0/authorize
+    ?response_type=code
+    &client_id=<client_id>
+    &scope=offline_access https://storage.azure.com/.default
+    &redirect_uri=http://localhost:8000/auth_flow
+    &response_mode=query
+    &state=1234
+
+2. POST https://login.microsoftonline.com/<tenant_id>/oauth2/v2.0/token
+client_id:<client_id>
+code:<code obtained from previous request>
+redirect_uri:http://localhost:8000/auth_flow
+grant_type:authorization_code
+client_secret:<client_secret>
+
 ### Installing the connector
 From this connector directory, run:
 ```bash

airbyte_source_azure_blob_storage-0.4.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+source_azure_blob_storage/__init__.py,sha256=dUrGWNqeIfK3e4yI6dgpIzSi8d03kItsUWCHQcmxXlg,281
+source_azure_blob_storage/config.py,sha256=w7hEWeY8cnNJ_jtAr-ZzkhyZ7pAxI6sQLUi2BG75jGQ,3949
+source_azure_blob_storage/config_migrations.py,sha256=_Dq-N4T6xTL2HpDmYIr-5K5CKCudUmEjA3x9wUdcHMg,2712
+source_azure_blob_storage/legacy_config_transformer.py,sha256=yiS4GgjaVvJtedmweAkq-eEdPG0RWWAAr0T9UV4RmOQ,1306
+source_azure_blob_storage/run.py,sha256=hUxQJCBSS_pzGzSE92Fum5C9EY7cU4yP3u-KefiC9cI,1890
+source_azure_blob_storage/source.py,sha256=lHyvHaEghSgxs1zK67o_x6viSVZ3oZOJca2Y-owI9J8,3456
+source_azure_blob_storage/stream_reader.py,sha256=YpPYv5l2eLRtxTNTLGQ6LCOHtwDdmC9TRLBt9M7ZGTU,4227
+airbyte_source_azure_blob_storage-0.4.0.dist-info/METADATA,sha256=IMQQw4bC4D6p4FGYRUPV-DstwWu2_uic9YJoMPn9P7Y,6246
+airbyte_source_azure_blob_storage-0.4.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+airbyte_source_azure_blob_storage-0.4.0.dist-info/entry_points.txt,sha256=75v_DA_Xu0qr0eqtEXyh8sPCqcL9eXKWY8UwdST3ANE,79
+airbyte_source_azure_blob_storage-0.4.0.dist-info/RECORD,,

source_azure_blob_storage/config.py

@@ -2,11 +2,51 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
-from typing import Any, Dict, Optional
+from typing import Any, Dict, Literal, Optional, Union
 
 import dpath.util
 from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec
-from pydantic import AnyUrl, Field
+from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig
+from pydantic import AnyUrl, BaseModel, Field
+
+
+class Oauth2(BaseModel):
+    class Config(OneOfOptionConfig):
+        title = "Authenticate via Oauth2"
+        discriminator = "auth_type"
+
+    auth_type: Literal["oauth2"] = Field("oauth2", const=True)
+    tenant_id: str = Field(title="Tenant ID", description="Tenant ID of the Microsoft Azure Application user", airbyte_secret=True)
+    client_id: str = Field(
+        title="Client ID",
+        description="Client ID of your Microsoft developer application",
+        airbyte_secret=True,
+    )
+    client_secret: str = Field(
+        title="Client Secret",
+        description="Client Secret of your Microsoft developer application",
+        airbyte_secret=True,
+    )
+    refresh_token: str = Field(
+        title="Refresh Token",
+        description="Refresh Token of your Microsoft developer application",
+        airbyte_secret=True,
+    )
+
+
+class StorageAccountKey(BaseModel):
+    class Config(OneOfOptionConfig):
+        title = "Authenticate via Storage Account Key"
+        discriminator = "auth_type"
+
+    auth_type: Literal["storage_account_key"] = Field("storage_account_key", const=True)
+    azure_blob_storage_account_key: str = Field(
+        title="Azure Blob Storage account key",
+        description="The Azure blob storage account key.",
+        airbyte_secret=True,
+        examples=["Z8ZkZpteggFx394vm+PJHnGTvdRncaYS+JhLKdj789YNmD+iyGTnG+PV+POiuYNhBg/ACS+LKjd%4FG3FHGN12Nd=="],
+        order=3,
+    )
 
 
 class Config(AbstractFileBasedSpec):
@@ -25,11 +65,11 @@ class Config(AbstractFileBasedSpec):
         examples=["airbyte5storage"],
         order=2,
     )
-    azure_blob_storage_account_key: str = Field(
-        title="Azure Blob Storage account key",
-        description="The Azure blob storage account key.",
-        airbyte_secret=True,
-        examples=["Z8ZkZpteggFx394vm+PJHnGTvdRncaYS+JhLKdj789YNmD+iyGTnG+PV+POiuYNhBg/ACS+LKjd%4FG3FHGN12Nd=="],
+    credentials: Union[Oauth2, StorageAccountKey] = Field(
+        title="Authentication",
+        description="Credentials for connecting to the Azure Blob Storage",
+        discriminator="auth_type",
+        type="object",
         order=3,
     )
     azure_blob_storage_container_name: str = Field(

source_azure_blob_storage/config_migrations.py

@@ -0,0 +1,77 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+
+import logging
+from typing import Any, List, Mapping
+
+from airbyte_cdk.config_observation import create_connector_config_control_message
+from airbyte_cdk.entrypoint import AirbyteEntrypoint
+from airbyte_cdk.sources import Source
+
+logger = logging.getLogger("airbyte_logger")
+
+
+class MigrateCredentials:
+    """
+    This class stands for migrating the config azure_blob_storage_account_key inside object `credentials`
+    """
+
+    @classmethod
+    def should_migrate(cls, config: Mapping[str, Any]) -> bool:
+        return "credentials" not in config
+
+    @classmethod
+    def set_azure_blob_storage_account_key(cls, config: Mapping[str, Any]) -> Mapping[str, Any]:
+        config["credentials"] = {
+            "auth_type": "storage_account_key",
+            "azure_blob_storage_account_key": config.pop("azure_blob_storage_account_key"),
+        }
+        return config
+
+    @classmethod
+    def modify_and_save(cls, config_path: str, source: Source, config: Mapping[str, Any]) -> Mapping[str, Any]:
+        """
+        Modifies the configuration and then saves it back to the source.
+
+        Args:
+        - config_path (str): The path where the configuration is stored.
+        - source (Source): The data source.
+        - config (Mapping[str, Any]): The current configuration.
+
+        Returns:
+        - Mapping[str, Any]: The updated configuration.
+        """
+        migrated_config = cls.set_azure_blob_storage_account_key(config)
+        source.write_config(migrated_config, config_path)
+        return migrated_config
+
+    @classmethod
+    def emit_control_message(cls, migrated_config: Mapping[str, Any]) -> None:
+        """
+        Emits the control messages related to configuration migration.
+
+        Args:
+        - migrated_config (Mapping[str, Any]): The migrated configuration.
+        """
+        print(create_connector_config_control_message(migrated_config).json(exclude_unset=True))
+
+    @classmethod
+    def migrate(cls, args: List[str], source: Source) -> None:
+        """
+        Orchestrates the configuration migration process.
+
+        It first checks if the `--config` argument is provided, and if so,
+        determines whether migration is needed, and then performs the migration
+        if required.
+
+        Args:
+        - args (List[str]): List of command-line arguments.
+        - source (Source): The data source.
+        """
+        config_path = AirbyteEntrypoint(source).extract_config(args)
+        if config_path:
+            config = source.read_config(config_path)
+            if cls.should_migrate(config):
+                cls.emit_control_message(cls.modify_and_save(config_path, source, config))

source_azure_blob_storage/run.py

@@ -10,6 +10,7 @@ from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch
 from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteTraceMessage, TraceType, Type
 from airbyte_cdk.sources.file_based.stream.cursor import DefaultFileBasedCursor
 from source_azure_blob_storage import Config, SourceAzureBlobStorage, SourceAzureBlobStorageStreamReader
+from source_azure_blob_storage.config_migrations import MigrateCredentials
 
 
 def run():
@@ -26,6 +27,7 @@ def run():
             SourceAzureBlobStorage.read_state(state_path) if catalog_path else None,
             cursor_cls=DefaultFileBasedCursor,
         )
+        MigrateCredentials.migrate(sys.argv[1:], source)
     except Exception:
         print(
             AirbyteMessage(

source_azure_blob_storage/source.py

@@ -5,7 +5,9 @@
 from typing import Any, Mapping
 
 from airbyte_cdk.config_observation import emit_configuration_as_airbyte_control_message
+from airbyte_cdk.sources.declarative.models import OAuthConfigSpecification
 from airbyte_cdk.sources.file_based.file_based_source import FileBasedSource
+from airbyte_protocol.models import AdvancedAuth, ConnectorSpecification
 
 from .legacy_config_transformer import LegacyConfigTransformer
 
@@ -28,3 +30,43 @@ class SourceAzureBlobStorage(FileBasedSource):
     @staticmethod
     def _is_v1_config(config: Mapping[str, Any]) -> bool:
         return "streams" in config
+
+    def spec(self, *args: Any, **kwargs: Any) -> ConnectorSpecification:
+        """
+        Returns the specification describing what fields can be configured by a user when setting up a file-based source.
+        """
+
+        return ConnectorSpecification(
+            documentationUrl=self.spec_class.documentation_url(),
+            connectionSpecification=self.spec_class.schema(),
+            advanced_auth=AdvancedAuth(
+                auth_flow_type="oauth2.0",
+                predicate_key=["credentials", "auth_type"],
+                predicate_value="oauth2",
+                oauth_config_specification=OAuthConfigSpecification(
+                    complete_oauth_output_specification={
+                        "type": "object",
+                        "additionalProperties": False,
+                        "properties": {"refresh_token": {"type": "string", "path_in_connector_config": ["credentials", "refresh_token"]}},
+                    },
+                    complete_oauth_server_input_specification={
+                        "type": "object",
+                        "additionalProperties": False,
+                        "properties": {"client_id": {"type": "string"}, "client_secret": {"type": "string"}},
+                    },
+                    complete_oauth_server_output_specification={
+                        "type": "object",
+                        "additionalProperties": False,
+                        "properties": {
+                            "client_id": {"type": "string", "path_in_connector_config": ["credentials", "client_id"]},
+                            "client_secret": {"type": "string", "path_in_connector_config": ["credentials", "client_secret"]},
+                        },
+                    },
+                    oauth_user_input_from_connector_config_specification={
+                        "type": "object",
+                        "additionalProperties": False,
+                        "properties": {"tenant_id": {"type": "string", "path_in_connector_config": ["credentials", "tenant_id"]}},
+                    },
+                ),
+            ),
+        )

source_azure_blob_storage/stream_reader.py

@@ -1,20 +1,35 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 
 import logging
-from contextlib import contextmanager
 from io import IOBase
-from typing import Iterable, List, Optional
+from typing import Iterable, List, Optional, Union
 
 import pytz
 from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
+from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator
+from azure.core.credentials import AccessToken
 from azure.storage.blob import BlobServiceClient, ContainerClient
 from smart_open import open
 
 from .config import Config
 
 
+class AzureOauth2Authenticator(Oauth2Authenticator):
+    """
+    Authenticator for Azure Blob Storage SDK to align with azure.core.credentials.TokenCredential protocol
+    """
+
+    def get_token(self, *args, **kwargs) -> AccessToken:
+        """Parent class handles Oauth Refresh token logic.
+        `expires_on` is ignored and set to year 2222 to align with protocol.
+        """
+        return AccessToken(token=self.get_access_token(), expires_on=7952342400)
+
+
 class SourceAzureBlobStorageStreamReader(AbstractFileBasedStreamReader):
+    _credentials = None
+
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self._config = None
@@ -36,14 +51,26 @@ class SourceAzureBlobStorageStreamReader(AbstractFileBasedStreamReader):
     @property
     def azure_container_client(self):
         return ContainerClient(
-            self.account_url,
-            container_name=self.config.azure_blob_storage_container_name,
-            credential=self.config.azure_blob_storage_account_key,
+            self.account_url, container_name=self.config.azure_blob_storage_container_name, credential=self.azure_credentials
         )
 
     @property
     def azure_blob_service_client(self):
-        return BlobServiceClient(self.account_url, credential=self.config.azure_blob_storage_account_key)
+        return BlobServiceClient(self.account_url, credential=self._credentials)
+
+    @property
+    def azure_credentials(self) -> Union[str, AzureOauth2Authenticator]:
+        if not self._credentials:
+            if self.config.credentials.auth_type == "storage_account_key":
+                self._credentials = self.config.credentials.azure_blob_storage_account_key
+            else:
+                self._credentials = AzureOauth2Authenticator(
+                    token_refresh_endpoint=f"https://login.microsoftonline.com/{self.config.credentials.tenant_id}/oauth2/v2.0/token",
+                    client_id=self.config.credentials.client_id,
+                    client_secret=self.config.credentials.client_secret,
+                    refresh_token=self.config.credentials.refresh_token,
+                )
+        return self._credentials
 
     def get_matching_files(
         self,

airbyte_source_azure_blob_storage-0.3.5.dist-info/RECORD

@@ -1,10 +0,0 @@
-source_azure_blob_storage/__init__.py,sha256=dUrGWNqeIfK3e4yI6dgpIzSi8d03kItsUWCHQcmxXlg,281
-source_azure_blob_storage/config.py,sha256=fg1fitDbsX-LXtPGBmG3bfu-jb0XoFYqHxGWsYY6Uy8,2499
-source_azure_blob_storage/legacy_config_transformer.py,sha256=yiS4GgjaVvJtedmweAkq-eEdPG0RWWAAr0T9UV4RmOQ,1306
-source_azure_blob_storage/run.py,sha256=xam3N4LxPECFb0HTpJKIrmr8R_bVJmKiqEN4sluK4yk,1758
-source_azure_blob_storage/source.py,sha256=dt6BmjpDsxjeSlm6BccfFD3NJblEmDk4gUiJnvjcU40,1186
-source_azure_blob_storage/stream_reader.py,sha256=HA9jNcsMKK-5TVlVelpV2JmjxSf0OfqxC0yBEUtdEe0,2940
-airbyte_source_azure_blob_storage-0.3.5.dist-info/METADATA,sha256=aobYZFqV7OVJs68wTuObNEYRPEd7zJ_NDElXom1-aY0,5536
-airbyte_source_azure_blob_storage-0.3.5.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-airbyte_source_azure_blob_storage-0.3.5.dist-info/entry_points.txt,sha256=75v_DA_Xu0qr0eqtEXyh8sPCqcL9eXKWY8UwdST3ANE,79
-airbyte_source_azure_blob_storage-0.3.5.dist-info/RECORD,,