airbyte-source-google-drive 0.2.0.dev202502031727__py3-none-any.whl → 0.2.0.dev202502052009__py3-none-any.whl

{airbyte_source_google_drive-0.2.0.dev202502031727.dist-info → airbyte_source_google_drive-0.2.0.dev202502052009.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: airbyte-source-google-drive
-Version: 0.2.0.dev202502031727
+Version: 0.2.0.dev202502052009
 Summary: Source implementation for Google Drive.
 License: ELv2
 Author: Airbyte
@@ -10,7 +10,7 @@ Classifier: License :: Other/Proprietary License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: airbyte-cdk[file-based] (==6.26.0.dev04106)
+Requires-Dist: airbyte-cdk[file-based] (==6.26.0.dev04108)
 Requires-Dist: google-api-python-client (==2.104.0)
 Requires-Dist: google-api-python-client-stubs (==1.18.0)
 Requires-Dist: google-auth-httplib2 (==0.1.1)

{airbyte_source_google_drive-0.2.0.dev202502031727.dist-info → airbyte_source_google_drive-0.2.0.dev202502052009.dist-info}/RECORD

@@ -2,10 +2,10 @@ source_google_drive/__init__.py,sha256=SAgPIoGpjdHTY7mvf__A99FiElBh3WvdKSHLstg-B
 source_google_drive/exceptions.py,sha256=6sBxbdZhj0CARkctfBaNgGHB_GGcFUkY4cnMUWyOruM,268
 source_google_drive/run.py,sha256=5AQd906FbSGaNZj0udrr-VKmiSS4nsor7F7noMjsdi0,677
 source_google_drive/source.py,sha256=OyKTjPg80KR8yZiYiC3YSi6E6egothtAn3YAHAS6Q6A,2839
-source_google_drive/spec.py,sha256=YPOmOInQfmwHeB6Ay2z78LNvHsggncwU1bchQOsAZSk,4268
-source_google_drive/stream_reader.py,sha256=h4p3aafE8U9BqYgRBGXuADEhm4NCGMqYl2ExnQkj6bU,20266
+source_google_drive/spec.py,sha256=-WkA2zGuQtf3G7uK8uq9BnimUlQh0s3vsqROmIHOgzI,4718
+source_google_drive/stream_reader.py,sha256=jKMibQGjpP-9raJ557TqbOg6iGVuPYv9vamStwXDOCQ,20176
 source_google_drive/utils.py,sha256=ewR-kBKLmtD-s7zqCfGECfzWYF43tpQdscAQIlUEkR8,1022
-airbyte_source_google_drive-0.2.0.dev202502031727.dist-info/METADATA,sha256=rMydjQTVW8_21SbChXDmp3i-LbEYiZi1OwOPvEKHh_k,5536
-airbyte_source_google_drive-0.2.0.dev202502031727.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
-airbyte_source_google_drive-0.2.0.dev202502031727.dist-info/entry_points.txt,sha256=YgpJf0nA5Mn0B7YC9VOFI847vz1jI6U4q7BeLUOXa54,67
-airbyte_source_google_drive-0.2.0.dev202502031727.dist-info/RECORD,,
+airbyte_source_google_drive-0.2.0.dev202502052009.dist-info/METADATA,sha256=fiVV_jIGTrIB43tzlnCv3BAkJEbJqr8BYtB25-vIUBU,5536
+airbyte_source_google_drive-0.2.0.dev202502052009.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
+airbyte_source_google_drive-0.2.0.dev202502052009.dist-info/entry_points.txt,sha256=YgpJf0nA5Mn0B7YC9VOFI847vz1jI6U4q7BeLUOXa54,67
+airbyte_source_google_drive-0.2.0.dev202502052009.dist-info/RECORD,,

source_google_drive/spec.py

@@ -1,7 +1,9 @@
 #
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
-
+import uuid
+from datetime import datetime
+from enum import Enum
 from typing import Any, Dict, Literal, Optional, Union
 
 import dpath.util
@@ -11,22 +13,40 @@ from airbyte_cdk import OneOfOptionConfig
 from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import (
     AbstractFileBasedSpec,
     DeliverRawFiles,
+    DeliverRecords,
 )
 from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import (
-    DeliverRecords as DeliverRecordsBase,
+    DeliverPermissions as DeliverPermissionsBase,
 )
 
 
-class DeliverRecords(DeliverRecordsBase):
-    # Overriding to make visible with airbyte_hidden=False
-    sync_acl_permissions: bool = Field(
-        title="Include ACL Permissions",
-        description="Joins Document allowlists to each stream.",
-        default=False,
-        airbyte_hidden=False,
-        order=0,
-    )
-    domain: Optional[str] = Field(title="Domain", description="The domain of the identities.", airbyte_hidden=False, order=1)
+class RemoteIdentityType(Enum):
+    USER = "user"
+    GROUP = "group"
+
+
+class RemoteIdentity(BaseModel):
+    id: uuid.UUID
+    remote_id: str
+    parent_id: str | None = None
+    name: str | None = None
+    description: str | None = None
+    email_address: str | None = None
+    member_email_addresses: list[str] | None = None
+    type: RemoteIdentityType
+    modified_at: datetime
+
+
+class RemotePermissions(BaseModel):
+    id: str
+    file_path: str
+    allowed_identity_remote_ids: list[str] | None = None
+    denied_identity_remote_ids: list[str] | None = None
+    publicly_accessible: bool = False
+
+
+class DeliverPermissions(DeliverPermissionsBase):
+    domain: Optional[str] = Field(title="Domain", description="The Google domain of the identities.", airbyte_hidden=False, order=1)
 
 
 class OAuthCredentials(BaseModel):
@@ -77,7 +97,7 @@ class SourceGoogleDriveSpec(AbstractFileBasedSpec, BaseModel):
         pattern_descriptor="https://drive.google.com/drive/folders/MY-FOLDER-ID",
     )
 
-    delivery_method: DeliverRecords | DeliverRawFiles = Field(
+    delivery_method: DeliverRecords | DeliverRawFiles | DeliverPermissions = Field(
         title="Delivery Method",
         discriminator="delivery_type",
         type="object",

source_google_drive/stream_reader.py

@@ -18,14 +18,13 @@ from googleapiclient.discovery import build
 from googleapiclient.http import MediaIoBaseDownload
 
 from airbyte_cdk import AirbyteTracedException, FailureType
-from airbyte_cdk.sources.file_based.config.permissions import RemoteFileIdentity, RemoteFileIdentityType, RemoteFilePermissions
 from airbyte_cdk.sources.file_based.exceptions import FileSizeLimitError
 from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
 from source_google_drive.utils import get_folder_id
 
 from .exceptions import ErrorDownloadingFile, ErrorFetchingMetadata
-from .spec import SourceGoogleDriveSpec
+from .spec import RemoteIdentity, RemoteIdentityType, RemotePermissions, SourceGoogleDriveSpec
 
 
 FOLDER_MIME_TYPE = "application/vnd.google-apps.folder"
@@ -64,11 +63,8 @@ PUBLIC_PERMISSION_IDS = [
     "domainWithLink",
 ]
 
-PERMISSIONS_API_SCOPES = [
-    "https://www.googleapis.com/auth/drive",
-    "https://www.googleapis.com/auth/drive.readonly",
-    "https://www.googleapis.com/auth/drive.metadata.readonly",
-    "https://www.googleapis.com/auth/drive.file",
+
+DRIVE_SERVICE_SCOPES = [
     "https://www.googleapis.com/auth/admin.directory.group.readonly",
     "https://www.googleapis.com/auth/admin.directory.group.member.readonly",
     "https://www.googleapis.com/auth/admin.directory.user.readonly",
@@ -92,6 +88,7 @@ class SourceGoogleDriveStreamReader(AbstractFileBasedStreamReader):
     def __init__(self):
         super().__init__()
         self._drive_service = None
+        self._directory_service = None
 
     @property
     def config(self) -> SourceGoogleDriveSpec:
@@ -111,32 +108,41 @@ class SourceGoogleDriveStreamReader(AbstractFileBasedStreamReader):
         assert isinstance(value, SourceGoogleDriveSpec)
         self._config = value
 
-    @property
-    def google_drive_service(self):
+    def _build_google_service(self, service_name: str, version: str, scopes: List[str] = None):
         if self.config is None:
             # We shouldn't hit this; config should always get set before attempting to
             # list or read files.
-            raise ValueError("Source config is missing; cannot create the Google Drive client.")
+            raise ValueError(f"Source config is missing; cannot create the Google {service_name} client.")
         try:
-            if self._drive_service is None:
-                if self.config.credentials.auth_type == "Client":
-                    creds = credentials.Credentials.from_authorized_user_info(self.config.credentials.dict())
-                else:
-                    scopes = PERMISSIONS_API_SCOPES if self.sync_acl_permissions() else None
-                    creds = service_account.Credentials.from_service_account_info(
-                        json.loads(self.config.credentials.service_account_info), scopes=scopes
-                    )
-                self._drive_service = build("drive", "v3", credentials=creds)
+            if self.config.credentials.auth_type == "Client":
+                creds = credentials.Credentials.from_authorized_user_info(self.config.credentials.dict())
+            else:
+                creds = service_account.Credentials.from_service_account_info(
+                    json.loads(self.config.credentials.service_account_info), scopes=scopes
+                )
+            google_service = build(service_name, version, credentials=creds)
         except Exception as e:
             raise AirbyteTracedException(
                 internal_message=str(e),
-                message="Could not authenticate with Google Drive. Please check your credentials.",
+                message=f"Could not authenticate with Google {service_name}. Please check your credentials.",
                 failure_type=FailureType.config_error,
                 exception=e,
             )
 
+        return google_service
+
+    @property
+    def google_drive_service(self):
+        if self._drive_service is None:
+            self._drive_service = self._build_google_service("drive", "v3")
         return self._drive_service
 
+    @property
+    def google_directory_service(self):
+        if self._directory_service is None:
+            self._directory_service = self._build_google_service("admin", "directory_v1", DRIVE_SERVICE_SCOPES)
+        return self._directory_service
+
     def get_matching_files(self, globs: List[str], prefix: Optional[str], logger: logging.Logger) -> Iterable[RemoteFile]:
         """
         Get all files matching the specified glob patterns.
@@ -310,7 +316,7 @@ class SourceGoogleDriveStreamReader(AbstractFileBasedStreamReader):
         except Exception as e:
             raise ErrorDownloadingFile(f"There was an error while trying to download the file {file.uri}: {str(e)}")
 
-    def get_file_permissions(self, file_id: str, file_name: str, logger: logging.Logger) -> Tuple[List[RemoteFileIdentity], bool]:
+    def get_file_permissions(self, file_id: str, file_name: str, logger: logging.Logger) -> Tuple[List[RemoteIdentity], bool]:
         """
         Retrieves the permissions of a file in Google Drive and checks for public access.
 
@@ -341,18 +347,17 @@ class SourceGoogleDriveStreamReader(AbstractFileBasedStreamReader):
                 if identity is not None:
                     remote_identities.append(identity)
 
-            logger.info(f"File {file_name} has {len(remote_identities)} valid permissions")
             return remote_identities, is_public
         except Exception as e:
             raise ErrorFetchingMetadata(f"An error occurred while retrieving file permissions: {str(e)}")
 
-    def _to_remote_file_identity(self, identity: dict[str, Any]) -> RemoteFileIdentity | None:
+    def _to_remote_file_identity(self, identity: dict[str, Any]) -> RemoteIdentity | None:
         if identity.get("id") in PUBLIC_PERMISSION_IDS:
             return None
         if identity.get("deleted") is True:
             return None
 
-        return RemoteFileIdentity(
+        return RemoteIdentity(
             modified_at=datetime.now(),
             id=uuid.uuid4(),
             remote_id=identity.get("emailAddress"),
@@ -364,7 +369,7 @@ class SourceGoogleDriveStreamReader(AbstractFileBasedStreamReader):
 
     def get_file_acl_permissions(self, file: GoogleDriveRemoteFile, logger: logging.Logger) -> Dict[str, Any]:
         remote_identities, is_public = self.get_file_permissions(file.id, file_name=file.uri, logger=logger)
-        return RemoteFilePermissions(
+        return RemotePermissions(
             id=file.id,
             file_path=file.uri,
             allowed_identity_remote_ids=[p.remote_id for p in remote_identities],
@@ -400,34 +405,34 @@ class SourceGoogleDriveStreamReader(AbstractFileBasedStreamReader):
     def load_identity_groups(self, logger: logging.Logger) -> Dict[str, Any]:
         domain = self.config.delivery_method.domain
         if not domain:
-            raise Exception("No domain was provided")
-        if self.google_drive_service is None or self.google_drive_service._http.credentials is None:
-            raise Exception("No auth found")
+            logger.info("No domain provided. Trying to fetch identities from the user workspace.")
+            api_args = {"customer": "my_customer"}
+        else:
+            api_args = {"domain": domain}
 
-        directory_service = build("admin", "directory_v1", credentials=self.google_drive_service._http.credentials)
-        users_api = directory_service.users()
-        groups_api = directory_service.groups()
-        members_api = directory_service.members()
+        users_api = self.google_directory_service.users()
+        groups_api = self.google_directory_service.groups()
+        members_api = self.google_directory_service.members()
 
-        for user in self._get_looping_google_api_list_response(users_api, "users", {"domain": domain}, logger):
-            rfp = RemoteFileIdentity(
+        for user in self._get_looping_google_api_list_response(users_api, "users", args=api_args, logger=logger):
+            rfp = RemoteIdentity(
                 id=uuid.uuid4(),
                 remote_id=user["primaryEmail"],
                 name=user["name"]["fullName"] if user["name"] is not None else None,
                 email_address=user["primaryEmail"],
                 member_email_addresses=[x["address"] for x in user["emails"]],
-                type=RemoteFileIdentityType.USER,
+                type=RemoteIdentityType.USER,
                 modified_at=datetime_now(),
             )
             yield rfp.dict()
 
-        for group in self._get_looping_google_api_list_response(groups_api, "groups", {"domain": domain}, logger):
-            rfp = RemoteFileIdentity(
+        for group in self._get_looping_google_api_list_response(groups_api, "groups", args=api_args, logger=logger):
+            rfp = RemoteIdentity(
                 id=uuid.uuid4(),
                 remote_id=group["email"],
                 name=group["name"],
                 email_address=group["email"],
-                type=RemoteFileIdentityType.GROUP,
+                type=RemoteIdentityType.GROUP,
                 modified_at=datetime_now(),
             )