airbyte-source-google-drive 0.1.0rc1.dev202501201950__py3-none-any.whl → 0.2.0.dev202501241656__py3-none-any.whl

{airbyte_source_google_drive-0.1.0rc1.dev202501201950.dist-info → airbyte_source_google_drive-0.2.0.dev202501241656.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: airbyte-source-google-drive
-Version: 0.1.0rc1.dev202501201950
+Version: 0.2.0.dev202501241656
 Summary: Source implementation for Google Drive.
 License: ELv2
 Author: Airbyte
@@ -10,7 +10,7 @@ Classifier: License :: Other/Proprietary License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: airbyte-cdk[file-based] (>=6.18.2,<7.0.0)
+Requires-Dist: airbyte-cdk[file-based] (==6.26.0.dev04100)
 Requires-Dist: google-api-python-client (==2.104.0)
 Requires-Dist: google-api-python-client-stubs (==1.18.0)
 Requires-Dist: google-auth-httplib2 (==0.1.1)

airbyte_source_google_drive-0.2.0.dev202501241656.dist-info/RECORD

@@ -0,0 +1,11 @@
+source_google_drive/__init__.py,sha256=SAgPIoGpjdHTY7mvf__A99FiElBh3WvdKSHLstg-BUE,134
+source_google_drive/exceptions.py,sha256=6sBxbdZhj0CARkctfBaNgGHB_GGcFUkY4cnMUWyOruM,268
+source_google_drive/run.py,sha256=5AQd906FbSGaNZj0udrr-VKmiSS4nsor7F7noMjsdi0,677
+source_google_drive/source.py,sha256=OyKTjPg80KR8yZiYiC3YSi6E6egothtAn3YAHAS6Q6A,2839
+source_google_drive/spec.py,sha256=krZ1qRjUP7vV4JLGMgggMkP4kizGG4uj8mX0S0CUwU8,4706
+source_google_drive/stream_reader.py,sha256=8Ibh4vXdPElHfI4KgeBf2HqkQMMVWrV6shLGBdYXUu4,17008
+source_google_drive/utils.py,sha256=1Fe3J4KXI1iIf4rklgLoR4p1xkCzLQI7zj5dkyi4Vt0,942
+airbyte_source_google_drive-0.2.0.dev202501241656.dist-info/METADATA,sha256=3v6W_Q5H627SdajGIzK_u4r2llp7Uc9SAI700QMkOW4,5536
+airbyte_source_google_drive-0.2.0.dev202501241656.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
+airbyte_source_google_drive-0.2.0.dev202501241656.dist-info/entry_points.txt,sha256=YgpJf0nA5Mn0B7YC9VOFI847vz1jI6U4q7BeLUOXa54,67
+airbyte_source_google_drive-0.2.0.dev202501241656.dist-info/RECORD,,

source_google_drive/spec.py

@@ -2,14 +2,51 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
-
+import uuid
+from datetime import datetime
 from typing import Any, Dict, Literal, Union
 
 import dpath.util
 from pydantic.v1 import BaseModel, Field
 
 from airbyte_cdk import OneOfOptionConfig
-from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec, DeliverRawFiles, DeliverRecords
+from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import (
+    AbstractFileBasedSpec,
+    DeliverRawFiles,
+)
+from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import (
+    DeliverRecords as DeliverRecordsBase,
+)
+
+
+class RemoteFileIdentity(BaseModel):
+    id: uuid.UUID
+    remote_id: str
+    parent_id: str | None = None
+    name: str | None = None
+    description: str | None = None
+    email_address: str | None = None
+    member_email_addresses: list[str] | None = None
+    type: str
+    modified_at: datetime
+
+
+class RemoteFileMetadata(BaseModel):
+    id: str
+    file_path: str
+    allowed_identity_remote_ids: list[str] | None = None
+    denied_identity_remote_ids: list[str] | None = None
+    publicly_accessible: bool = False
+
+
+class DeliverRecords(DeliverRecordsBase):
+    # Overriding to make visible with airbyte_hidden=False
+    sync_metadata: bool = Field(
+        title="Make stream sync files metadata",
+        description="If enabled, streams will sync files metadata instead of files data.",
+        default=False,
+        airbyte_hidden=False,
+    )
 
 
 class OAuthCredentials(BaseModel):

source_google_drive/stream_reader.py

@@ -6,10 +6,11 @@
 import io
 import json
 import logging
+import uuid
 from datetime import datetime
 from io import IOBase
 from os.path import getsize
-from typing import Dict, Iterable, List, Optional, Set
+from typing import Any, Dict, Iterable, List, Optional, Set, Tuple
 
 from google.oauth2 import credentials, service_account
 from googleapiclient.discovery import build
@@ -22,7 +23,7 @@ from airbyte_cdk.sources.file_based.remote_file import RemoteFile
 from source_google_drive.utils import get_folder_id
 
 from .exceptions import ErrorDownloadingFile, ErrorFetchingMetadata
-from .spec import SourceGoogleDriveSpec
+from .spec import RemoteFileIdentity, RemoteFileMetadata, SourceGoogleDriveSpec
 
 
 FOLDER_MIME_TYPE = "application/vnd.google-apps.folder"
@@ -54,6 +55,20 @@ DOWNLOADABLE_DOCUMENTS_MIME_TYPES = {
     GOOGLE_DRAWING_MIME_TYPE: {EXPORT_MEDIA_MIME_TYPE_KEY: EXPORT_MEDIA_MIME_TYPE_PDF, DOCUMENT_FILE_EXTENSION_KEY: ".pdf"},
 }
 
+PUBLIC_PERMISSION_IDS = [
+    "anyoneWithLink",
+    "anyoneCanFind",
+    "domainCanFind",
+    "domainWithLink",
+]
+
+PERMISSIONS_API_SCOPES = [
+    "https://www.googleapis.com/auth/drive",
+    "https://www.googleapis.com/auth/admin.directory.group.readonly",
+    "https://www.googleapis.com/auth/admin.directory.group.member.readonly",
+    "https://www.googleapis.com/auth/admin.directory.user.readonly",
+]
+
 
 class GoogleDriveRemoteFile(RemoteFile):
     id: str
@@ -98,7 +113,10 @@ class SourceGoogleDriveStreamReader(AbstractFileBasedStreamReader):
                 if self.config.credentials.auth_type == "Client":
                     creds = credentials.Credentials.from_authorized_user_info(self.config.credentials.dict())
                 else:
-                    creds = service_account.Credentials.from_service_account_info(json.loads(self.config.credentials.service_account_info))
+                    scopes = PERMISSIONS_API_SCOPES if self.sync_metadata() else None
+                    creds = service_account.Credentials.from_service_account_info(
+                        json.loads(self.config.credentials.service_account_info), scopes=scopes
+                    )
                 self._drive_service = build("drive", "v3", credentials=creds)
         except Exception as e:
             raise AirbyteTracedException(
@@ -281,4 +299,65 @@ class SourceGoogleDriveStreamReader(AbstractFileBasedStreamReader):
             return {"file_url": absolute_file_path, "bytes": file_size, "file_relative_path": file_relative_path}
 
         except Exception as e:
-            raise ErrorDownloadingFile(f"There was an error while trying to download the file: {str(e)}")
+            raise ErrorDownloadingFile(f"There was an error while trying to download the file {file.uri}: {str(e)}")
+
+    def get_file_permissions(self, file_id: str, file_name: str, logger: logging.Logger) -> Tuple[List[RemoteFileIdentity], bool]:
+        """
+        Retrieves the permissions of a file in Google Drive and checks for public access.
+
+        Args:
+            file_id (str): The file to get permissions for.
+            file_name (str): The name of the file to get permissions for.
+            logger (logging.Logger): Logger for debugging and information.
+
+        Returns:
+            Tuple(List[RemoteFileIdentity], boolean): A list of RemoteFileIdentity objects containing permission details.
+        """
+        try:
+            request = self.google_drive_service.permissions().list(
+                fileId=file_id,
+                fields="permissions, permissions/role, permissions/type, permissions/id, permissions/emailAddress",
+                supportsAllDrives=True,
+            )
+            response = request.execute()
+            permissions = response.get("permissions", [])
+            is_public = False
+
+            remote_identities = []
+
+            for p in permissions:
+                identity = self._to_remote_file_identity(p)
+                if p.get("id") in PUBLIC_PERMISSION_IDS:
+                    is_public = True
+                if identity is not None:
+                    remote_identities.append(identity)
+
+            logger.info(f"File {file_name} has {len(remote_identities)} valid permissions")
+            return remote_identities, is_public
+        except Exception as e:
+            raise ErrorFetchingMetadata(f"An error occurred while retrieving file permissions: {str(e)}")
+
+    def _to_remote_file_identity(self, identity: dict[str, Any]) -> RemoteFileIdentity | None:
+        if identity.get("id") in PUBLIC_PERMISSION_IDS:
+            return None
+        if identity.get("deleted") is True:
+            return None
+
+        return RemoteFileIdentity(
+            modified_at=datetime.now(),
+            id=uuid.uuid4(),
+            remote_id=identity.get("emailAddress"),
+            name=identity.get("name"),
+            email_address=identity.get("emailAddress"),
+            type=identity.get("type"),
+            description=None,
+        )
+
+    def get_file_metadata(self, file: GoogleDriveRemoteFile, logger: logging.Logger) -> Dict[str, Any]:
+        remote_identities, is_public = self.get_file_permissions(file.id, file_name=file.uri, logger=logger)
+        return RemoteFileMetadata(
+            id=file.id,
+            file_path=file.uri,
+            allowed_identity_remote_ids=[p.remote_id for p in remote_identities],
+            publicly_accessible=is_public,
+        ).dict(exclude_none=True)

airbyte_source_google_drive-0.1.0rc1.dev202501201950.dist-info/RECORD

@@ -1,11 +0,0 @@
-source_google_drive/__init__.py,sha256=SAgPIoGpjdHTY7mvf__A99FiElBh3WvdKSHLstg-BUE,134
-source_google_drive/exceptions.py,sha256=6sBxbdZhj0CARkctfBaNgGHB_GGcFUkY4cnMUWyOruM,268
-source_google_drive/run.py,sha256=5AQd906FbSGaNZj0udrr-VKmiSS4nsor7F7noMjsdi0,677
-source_google_drive/source.py,sha256=OyKTjPg80KR8yZiYiC3YSi6E6egothtAn3YAHAS6Q6A,2839
-source_google_drive/spec.py,sha256=W5tUpyt-T9JjxsN6WTgz1SHBcGJnXg__L9M-nsDWOlk,3690
-source_google_drive/stream_reader.py,sha256=-BF7GuxyEInv4SeLHZhFvJUq1yaS2P5kRAvdtKwZFm8,13720
-source_google_drive/utils.py,sha256=1Fe3J4KXI1iIf4rklgLoR4p1xkCzLQI7zj5dkyi4Vt0,942
-airbyte_source_google_drive-0.1.0rc1.dev202501201950.dist-info/METADATA,sha256=RdzBQ6hgFqKmjz3HUky4Orxvi8dpcfoqFoIVmJQJ-AU,5537
-airbyte_source_google_drive-0.1.0rc1.dev202501201950.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
-airbyte_source_google_drive-0.1.0rc1.dev202501201950.dist-info/entry_points.txt,sha256=YgpJf0nA5Mn0B7YC9VOFI847vz1jI6U4q7BeLUOXa54,67
-airbyte_source_google_drive-0.1.0rc1.dev202501201950.dist-info/RECORD,,