airbyte-source-google-drive 0.2.0.dev202501291717__py3-none-any.whl → 0.2.0.dev202502032236__py3-none-any.whl

{airbyte_source_google_drive-0.2.0.dev202501291717.dist-info → airbyte_source_google_drive-0.2.0.dev202502032236.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: airbyte-source-google-drive
-Version: 0.2.0.dev202501291717
+Version: 0.2.0.dev202502032236
 Summary: Source implementation for Google Drive.
 License: ELv2
 Author: Airbyte
@@ -10,7 +10,7 @@ Classifier: License :: Other/Proprietary License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: airbyte-cdk[file-based] (==6.26.0.dev04106)
+Requires-Dist: airbyte-cdk[file-based] (==6.26.0.dev04107)
 Requires-Dist: google-api-python-client (==2.104.0)
 Requires-Dist: google-api-python-client-stubs (==1.18.0)
 Requires-Dist: google-auth-httplib2 (==0.1.1)

airbyte_source_google_drive-0.2.0.dev202502032236.dist-info/RECORD

@@ -0,0 +1,11 @@
+source_google_drive/__init__.py,sha256=SAgPIoGpjdHTY7mvf__A99FiElBh3WvdKSHLstg-BUE,134
+source_google_drive/exceptions.py,sha256=6sBxbdZhj0CARkctfBaNgGHB_GGcFUkY4cnMUWyOruM,268
+source_google_drive/run.py,sha256=5AQd906FbSGaNZj0udrr-VKmiSS4nsor7F7noMjsdi0,677
+source_google_drive/source.py,sha256=OyKTjPg80KR8yZiYiC3YSi6E6egothtAn3YAHAS6Q6A,2839
+source_google_drive/spec.py,sha256=-WkA2zGuQtf3G7uK8uq9BnimUlQh0s3vsqROmIHOgzI,4718
+source_google_drive/stream_reader.py,sha256=8F0jEKz1ElM2wqHxVR91RUJfCSDoLtMQUxbX5lFkEy0,20166
+source_google_drive/utils.py,sha256=ewR-kBKLmtD-s7zqCfGECfzWYF43tpQdscAQIlUEkR8,1022
+airbyte_source_google_drive-0.2.0.dev202502032236.dist-info/METADATA,sha256=Hfb2iDOlay912TAmyj3QZ9GqvyVstYZflM_J2lpf2Lk,5536
+airbyte_source_google_drive-0.2.0.dev202502032236.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
+airbyte_source_google_drive-0.2.0.dev202502032236.dist-info/entry_points.txt,sha256=YgpJf0nA5Mn0B7YC9VOFI847vz1jI6U4q7BeLUOXa54,67
+airbyte_source_google_drive-0.2.0.dev202502032236.dist-info/RECORD,,

source_google_drive/spec.py

@@ -1,7 +1,9 @@
 #
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
-
+import uuid
+from datetime import datetime
+from enum import Enum
 from typing import Any, Dict, Literal, Optional, Union
 
 import dpath.util
@@ -11,22 +13,40 @@ from airbyte_cdk import OneOfOptionConfig
 from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import (
     AbstractFileBasedSpec,
     DeliverRawFiles,
+    DeliverRecords,
 )
 from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import (
-    DeliverRecords as DeliverRecordsBase,
+    DeliverPermissions as DeliverPermissionsBase,
 )
 
 
-class DeliverRecords(DeliverRecordsBase):
-    # Overriding to make visible with airbyte_hidden=False
-    sync_acl_permissions: bool = Field(
-        title="Include ACL Permissions",
-        description="Joins Document allowlists to each stream.",
-        default=False,
-        airbyte_hidden=False,
-        order=0,
-    )
-    domain: Optional[str] = Field(title="Domain", description="The domain of the identities.", airbyte_hidden=False, order=1)
+class RemoteIdentityType(Enum):
+    USER = "user"
+    GROUP = "group"
+
+
+class RemoteIdentity(BaseModel):
+    id: uuid.UUID
+    remote_id: str
+    parent_id: str | None = None
+    name: str | None = None
+    description: str | None = None
+    email_address: str | None = None
+    member_email_addresses: list[str] | None = None
+    type: RemoteIdentityType
+    modified_at: datetime
+
+
+class RemotePermissions(BaseModel):
+    id: str
+    file_path: str
+    allowed_identity_remote_ids: list[str] | None = None
+    denied_identity_remote_ids: list[str] | None = None
+    publicly_accessible: bool = False
+
+
+class DeliverPermissions(DeliverPermissionsBase):
+    domain: Optional[str] = Field(title="Domain", description="The Google domain of the identities.", airbyte_hidden=False, order=1)
 
 
 class OAuthCredentials(BaseModel):
@@ -77,7 +97,7 @@ class SourceGoogleDriveSpec(AbstractFileBasedSpec, BaseModel):
         pattern_descriptor="https://drive.google.com/drive/folders/MY-FOLDER-ID",
     )
 
-    delivery_method: DeliverRecords | DeliverRawFiles = Field(
+    delivery_method: DeliverRecords | DeliverRawFiles | DeliverPermissions = Field(
         title="Delivery Method",
         discriminator="delivery_type",
         type="object",

source_google_drive/stream_reader.py

@@ -18,14 +18,13 @@ from googleapiclient.discovery import build
 from googleapiclient.http import MediaIoBaseDownload
 
 from airbyte_cdk import AirbyteTracedException, FailureType
-from airbyte_cdk.sources.file_based.config.permissions import RemoteFileIdentity, RemoteFileIdentityType, RemoteFilePermissions
 from airbyte_cdk.sources.file_based.exceptions import FileSizeLimitError
 from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
 from source_google_drive.utils import get_folder_id
 
 from .exceptions import ErrorDownloadingFile, ErrorFetchingMetadata
-from .spec import SourceGoogleDriveSpec
+from .spec import RemoteIdentity, RemoteIdentityType, RemotePermissions, SourceGoogleDriveSpec
 
 
 FOLDER_MIME_TYPE = "application/vnd.google-apps.folder"
@@ -122,7 +121,7 @@ class SourceGoogleDriveStreamReader(AbstractFileBasedStreamReader):
                 if self.config.credentials.auth_type == "Client":
                     creds = credentials.Credentials.from_authorized_user_info(self.config.credentials.dict())
                 else:
-                    scopes = PERMISSIONS_API_SCOPES if self.sync_acl_permissions() else None
+                    scopes = PERMISSIONS_API_SCOPES if self.include_identities_stream() else None
                     creds = service_account.Credentials.from_service_account_info(
                         json.loads(self.config.credentials.service_account_info), scopes=scopes
                     )
@@ -137,73 +136,6 @@ class SourceGoogleDriveStreamReader(AbstractFileBasedStreamReader):
 
         return self._drive_service
 
-    def _get_looping_google_api_list_response(
-        self, service: Any, key: str, args: dict[str, Any], logger: logging.Logger
-    ) -> Iterator[dict[str, Any]]:
-        try:
-            looping = True
-            next_page_token: str | None = None
-            while looping:
-                rsp = service.list(pageToken=next_page_token, **args).execute()
-                next_page_token = rsp.get("nextPageToken")
-                items: list[dict[str, Any]] = rsp.get(key)
-
-                if items is None or len(items) == 0:
-                    looping = False
-                    break
-
-                if rsp.get("nextPageToken") is None:
-                    looping = False
-                else:
-                    next_page_token = rsp.get("nextPageToken")
-
-                for item in items:
-                    yield item
-        except Exception as e:
-            logger.error(f"There was an error listing {key} with {args}: {str(e)}")
-            raise e
-
-    def load_identity_groups(self, logger: logging.Logger) -> Dict[str, Any]:
-        domain = self.config.delivery_method.domain
-        if not domain:
-            raise Exception("No domain was provided")
-        if self.google_drive_service is None or self.google_drive_service._http.credentials is None:
-            raise Exception("No auth found")
-
-        directory_service = build("admin", "directory_v1", credentials=self.google_drive_service._http.credentials)
-        users_api = directory_service.users()
-        groups_api = directory_service.groups()
-        members_api = directory_service.members()
-
-        # here is failing
-        for user in self._get_looping_google_api_list_response(users_api, "users", {"domain": domain}, logger):
-            rfp = RemoteFileIdentity(
-                id=uuid.uuid4(),
-                remote_id=user["primaryEmail"],
-                name=user["name"]["fullName"] if user["name"] is not None else None,
-                email_address=user["primaryEmail"],
-                member_email_addresses=[x["address"] for x in user["emails"]],
-                type=RemoteFileIdentityType.USER,
-                modified_at=datetime_now(),
-            )
-            yield rfp.dict()
-
-        for group in self._get_looping_google_api_list_response(groups_api, "groups", {"domain": domain}, logger):
-            rfp = RemoteFileIdentity(
-                id=uuid.uuid4(),
-                remote_id=group["email"],
-                name=group["name"],
-                email_address=group["email"],
-                type=RemoteFileIdentityType.GROUP,
-                modified_at=datetime_now(),
-            )
-
-            for member in self._get_looping_google_api_list_response(members_api, "members", {"groupKey": group["id"]}, logger):
-                rfp.member_email_addresses = rfp.member_email_addresses or []
-                rfp.member_email_addresses.append(member["email"])
-
-            yield rfp.dict()
-
     def get_matching_files(self, globs: List[str], prefix: Optional[str], logger: logging.Logger) -> Iterable[RemoteFile]:
         """
         Get all files matching the specified glob patterns.
@@ -377,7 +309,7 @@ class SourceGoogleDriveStreamReader(AbstractFileBasedStreamReader):
         except Exception as e:
             raise ErrorDownloadingFile(f"There was an error while trying to download the file {file.uri}: {str(e)}")
 
-    def get_file_permissions(self, file_id: str, file_name: str, logger: logging.Logger) -> Tuple[List[RemoteFileIdentity], bool]:
+    def get_file_permissions(self, file_id: str, file_name: str, logger: logging.Logger) -> Tuple[List[RemoteIdentity], bool]:
         """
         Retrieves the permissions of a file in Google Drive and checks for public access.
 
@@ -413,13 +345,13 @@ class SourceGoogleDriveStreamReader(AbstractFileBasedStreamReader):
         except Exception as e:
             raise ErrorFetchingMetadata(f"An error occurred while retrieving file permissions: {str(e)}")
 
-    def _to_remote_file_identity(self, identity: dict[str, Any]) -> RemoteFileIdentity | None:
+    def _to_remote_file_identity(self, identity: dict[str, Any]) -> RemoteIdentity | None:
         if identity.get("id") in PUBLIC_PERMISSION_IDS:
             return None
         if identity.get("deleted") is True:
             return None
 
-        return RemoteFileIdentity(
+        return RemoteIdentity(
             modified_at=datetime.now(),
             id=uuid.uuid4(),
             remote_id=identity.get("emailAddress"),
@@ -431,9 +363,75 @@ class SourceGoogleDriveStreamReader(AbstractFileBasedStreamReader):
 
     def get_file_acl_permissions(self, file: GoogleDriveRemoteFile, logger: logging.Logger) -> Dict[str, Any]:
         remote_identities, is_public = self.get_file_permissions(file.id, file_name=file.uri, logger=logger)
-        return RemoteFilePermissions(
+        return RemotePermissions(
             id=file.id,
             file_path=file.uri,
             allowed_identity_remote_ids=[p.remote_id for p in remote_identities],
             publicly_accessible=is_public,
         ).dict(exclude_none=True)
+
+    def _get_looping_google_api_list_response(
+        self, service: Any, key: str, args: dict[str, Any], logger: logging.Logger
+    ) -> Iterator[dict[str, Any]]:
+        try:
+            looping = True
+            next_page_token: str | None = None
+            while looping:
+                rsp = service.list(pageToken=next_page_token, **args).execute()
+                next_page_token = rsp.get("nextPageToken")
+                items: list[dict[str, Any]] = rsp.get(key)
+
+                if items is None or len(items) == 0:
+                    looping = False
+                    break
+
+                if rsp.get("nextPageToken") is None:
+                    looping = False
+                else:
+                    next_page_token = rsp.get("nextPageToken")
+
+                for item in items:
+                    yield item
+        except Exception as e:
+            logger.error(f"There was an error listing {key} with {args}: {str(e)}")
+            raise e
+
+    def load_identity_groups(self, logger: logging.Logger) -> Dict[str, Any]:
+        domain = self.config.delivery_method.domain
+        if not domain:
+            raise Exception("No domain was provided")
+        if self.google_drive_service is None or self.google_drive_service._http.credentials is None:
+            raise Exception("No auth found")
+
+        directory_service = build("admin", "directory_v1", credentials=self.google_drive_service._http.credentials)
+        users_api = directory_service.users()
+        groups_api = directory_service.groups()
+        members_api = directory_service.members()
+
+        for user in self._get_looping_google_api_list_response(users_api, "users", {"domain": domain}, logger):
+            rfp = RemoteIdentity(
+                id=uuid.uuid4(),
+                remote_id=user["primaryEmail"],
+                name=user["name"]["fullName"] if user["name"] is not None else None,
+                email_address=user["primaryEmail"],
+                member_email_addresses=[x["address"] for x in user["emails"]],
+                type=RemoteIdentityType.USER,
+                modified_at=datetime_now(),
+            )
+            yield rfp.dict()
+
+        for group in self._get_looping_google_api_list_response(groups_api, "groups", {"domain": domain}, logger):
+            rfp = RemoteIdentity(
+                id=uuid.uuid4(),
+                remote_id=group["email"],
+                name=group["name"],
+                email_address=group["email"],
+                type=RemoteIdentityType.GROUP,
+                modified_at=datetime_now(),
+            )
+
+            for member in self._get_looping_google_api_list_response(members_api, "members", {"groupKey": group["id"]}, logger):
+                rfp.member_email_addresses = rfp.member_email_addresses or []
+                rfp.member_email_addresses.append(member["email"])
+
+            yield rfp.dict()

source_google_drive/utils.py

@@ -15,6 +15,8 @@ def get_folder_id(url_string: str) -> str:
         if parsed_url.scheme != "https" or parsed_url.netloc != "drive.google.com":
             raise ValueError("Folder URL has to be of the form https://drive.google.com/drive/folders/<folder_id>")
         path_segments = list(filter(None, parsed_url.path.split("/")))
+        if path_segments[-1] in ["my-drive", "home"]:
+            return "root"
         if path_segments[-2] != "folders" or len(path_segments) < 3:
             raise ValueError("Folder URL has to be of the form https://drive.google.com/drive/folders/<folder_id>")
         return path_segments[-1]

airbyte_source_google_drive-0.2.0.dev202501291717.dist-info/RECORD

@@ -1,11 +0,0 @@
-source_google_drive/__init__.py,sha256=SAgPIoGpjdHTY7mvf__A99FiElBh3WvdKSHLstg-BUE,134
-source_google_drive/exceptions.py,sha256=6sBxbdZhj0CARkctfBaNgGHB_GGcFUkY4cnMUWyOruM,268
-source_google_drive/run.py,sha256=5AQd906FbSGaNZj0udrr-VKmiSS4nsor7F7noMjsdi0,677
-source_google_drive/source.py,sha256=OyKTjPg80KR8yZiYiC3YSi6E6egothtAn3YAHAS6Q6A,2839
-source_google_drive/spec.py,sha256=YPOmOInQfmwHeB6Ay2z78LNvHsggncwU1bchQOsAZSk,4268
-source_google_drive/stream_reader.py,sha256=DClCknbFqAq1SZQQBJnWVpbIoJkfu7ZeLzLh0E0aE5U,20292
-source_google_drive/utils.py,sha256=1Fe3J4KXI1iIf4rklgLoR4p1xkCzLQI7zj5dkyi4Vt0,942
-airbyte_source_google_drive-0.2.0.dev202501291717.dist-info/METADATA,sha256=gXbq2ilBqoJCjwtdwqmSgFVL3RxFMlxgwUxODMZT4Yg,5536
-airbyte_source_google_drive-0.2.0.dev202501291717.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
-airbyte_source_google_drive-0.2.0.dev202501291717.dist-info/entry_points.txt,sha256=YgpJf0nA5Mn0B7YC9VOFI847vz1jI6U4q7BeLUOXa54,67
-airbyte_source_google_drive-0.2.0.dev202501291717.dist-info/RECORD,,