airbyte-source-google-drive 0.2.0.dev202501272304__tar.gz → 0.2.0.dev202502031727__tar.gz

{airbyte_source_google_drive-0.2.0.dev202501272304 → airbyte_source_google_drive-0.2.0.dev202502031727}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: airbyte-source-google-drive
-Version: 0.2.0.dev202501272304
+Version: 0.2.0.dev202502031727
 Summary: Source implementation for Google Drive.
 License: ELv2
 Author: Airbyte
@@ -10,7 +10,7 @@ Classifier: License :: Other/Proprietary License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: airbyte-cdk[file-based] (==6.26.0.dev04105)
+Requires-Dist: airbyte-cdk[file-based] (==6.26.0.dev04106)
 Requires-Dist: google-api-python-client (==2.104.0)
 Requires-Dist: google-api-python-client-stubs (==1.18.0)
 Requires-Dist: google-auth-httplib2 (==0.1.1)

{airbyte_source_google_drive-0.2.0.dev202501272304 → airbyte_source_google_drive-0.2.0.dev202502031727}/pyproject.toml

@@ -5,7 +5,7 @@ requires = [
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "0.2.0.dev202501272304"
+version = "0.2.0.dev202502031727"
 name = "airbyte-source-google-drive"
 description = "Source implementation for Google Drive."
 authors = [
@@ -31,7 +31,7 @@ google-api-python-client-stubs = "==1.18.0"
 extras = [
     "file-based",
 ]
-version = "6.26.0.dev04105"
+version = "6.26.0.dev04106"
 
 [tool.poetry.scripts]
 source-google-drive = "source_google_drive.run:run"

{airbyte_source_google_drive-0.2.0.dev202501272304 → airbyte_source_google_drive-0.2.0.dev202502031727}/source_google_drive/stream_reader.py

@@ -27,9 +27,6 @@ from source_google_drive.utils import get_folder_id
 from .exceptions import ErrorDownloadingFile, ErrorFetchingMetadata
 from .spec import SourceGoogleDriveSpec
 
-# remove this, just using while test credentials are prepared
-from .temp_mock import _get_looping_google_api_list_response
-
 
 FOLDER_MIME_TYPE = "application/vnd.google-apps.folder"
 GOOGLE_DOC_MIME_TYPE = "application/vnd.google-apps.document"
@@ -140,76 +137,6 @@ class SourceGoogleDriveStreamReader(AbstractFileBasedStreamReader):
 
         return self._drive_service
 
-    def _get_looping_google_api_list_response(
-        self, service: Any, key: str, args: dict[str, Any], logger: logging.Logger
-    ) -> Iterator[dict[str, Any]]:
-        try:
-            looping = True
-            next_page_token: str | None = None
-            while looping:
-                rsp = service.list(pageToken=next_page_token, **args).execute()
-                next_page_token = rsp.get("nextPageToken")
-                items: list[dict[str, Any]] = rsp.get(key)
-
-                if items is None or len(items) == 0:
-                    looping = False
-                    break
-
-                if rsp.get("nextPageToken") is None:
-                    looping = False
-                else:
-                    next_page_token = rsp.get("nextPageToken")
-
-                for item in items:
-                    yield item
-        except Exception as e:
-            logger.info(f"There was an error listing {key} with {args}: {str(e)}")
-            logger.info(f"Backing off to mocked data for development purposes")
-            yield from _get_looping_google_api_list_response(service, key, args)
-
-    def load_identity_groups(self, logger: logging.Logger) -> Dict[str, Any]:
-        domain = self.config.delivery_method.domain
-        if not domain:
-            raise Exception("No domain was provided")
-        if self.google_drive_service is None or self.google_drive_service._http.credentials is None:
-            raise Exception("No auth found")
-
-        directory_service = build("admin", "directory_v1", credentials=self.google_drive_service._http.credentials)
-        users_api = directory_service.users()
-        groups_api = directory_service.groups()
-        members_api = directory_service.members()
-
-        # here is failing
-        for user in self._get_looping_google_api_list_response(users_api, "users", {"domain": domain}, logger):
-            rfp = RemoteFileIdentity(
-                id=uuid.uuid4(),
-                remote_id=user["primaryEmail"],
-                name=user["name"]["fullName"] if user["name"] is not None else None,
-                email_address=user["primaryEmail"],
-                member_email_addresses=[x["address"] for x in user["emails"]],
-                type=RemoteFileIdentityType.USER,
-                modified_at=datetime_now(),
-            )
-            yield rfp.dict()
-
-        for group in self._get_looping_google_api_list_response(groups_api, "groups", {"domain": domain}, logger):
-            rfp = RemoteFileIdentity(
-                id=uuid.uuid4(),
-                remote_id=group["email"],
-                name=group["name"],
-                email_address=group["email"],
-                type=RemoteFileIdentityType.GROUP,
-                modified_at=datetime_now(),
-            )
-
-            for member in self._get_looping_google_api_list_response(members_api, "members", {"groupKey": group["id"]}, logger):
-                rfp.member_email_addresses = rfp.member_email_addresses or []
-                rfp.member_email_addresses.append(member["email"])
-
-            yield rfp.dict()
-
-        return ""
-
     def get_matching_files(self, globs: List[str], prefix: Optional[str], logger: logging.Logger) -> Iterable[RemoteFile]:
         """
         Get all files matching the specified glob patterns.
@@ -443,3 +370,69 @@ class SourceGoogleDriveStreamReader(AbstractFileBasedStreamReader):
             allowed_identity_remote_ids=[p.remote_id for p in remote_identities],
             publicly_accessible=is_public,
         ).dict(exclude_none=True)
+
+    def _get_looping_google_api_list_response(
+        self, service: Any, key: str, args: dict[str, Any], logger: logging.Logger
+    ) -> Iterator[dict[str, Any]]:
+        try:
+            looping = True
+            next_page_token: str | None = None
+            while looping:
+                rsp = service.list(pageToken=next_page_token, **args).execute()
+                next_page_token = rsp.get("nextPageToken")
+                items: list[dict[str, Any]] = rsp.get(key)
+
+                if items is None or len(items) == 0:
+                    looping = False
+                    break
+
+                if rsp.get("nextPageToken") is None:
+                    looping = False
+                else:
+                    next_page_token = rsp.get("nextPageToken")
+
+                for item in items:
+                    yield item
+        except Exception as e:
+            logger.error(f"There was an error listing {key} with {args}: {str(e)}")
+            raise e
+
+    def load_identity_groups(self, logger: logging.Logger) -> Dict[str, Any]:
+        domain = self.config.delivery_method.domain
+        if not domain:
+            raise Exception("No domain was provided")
+        if self.google_drive_service is None or self.google_drive_service._http.credentials is None:
+            raise Exception("No auth found")
+
+        directory_service = build("admin", "directory_v1", credentials=self.google_drive_service._http.credentials)
+        users_api = directory_service.users()
+        groups_api = directory_service.groups()
+        members_api = directory_service.members()
+
+        for user in self._get_looping_google_api_list_response(users_api, "users", {"domain": domain}, logger):
+            rfp = RemoteFileIdentity(
+                id=uuid.uuid4(),
+                remote_id=user["primaryEmail"],
+                name=user["name"]["fullName"] if user["name"] is not None else None,
+                email_address=user["primaryEmail"],
+                member_email_addresses=[x["address"] for x in user["emails"]],
+                type=RemoteFileIdentityType.USER,
+                modified_at=datetime_now(),
+            )
+            yield rfp.dict()
+
+        for group in self._get_looping_google_api_list_response(groups_api, "groups", {"domain": domain}, logger):
+            rfp = RemoteFileIdentity(
+                id=uuid.uuid4(),
+                remote_id=group["email"],
+                name=group["name"],
+                email_address=group["email"],
+                type=RemoteFileIdentityType.GROUP,
+                modified_at=datetime_now(),
+            )
+
+            for member in self._get_looping_google_api_list_response(members_api, "members", {"groupKey": group["id"]}, logger):
+                rfp.member_email_addresses = rfp.member_email_addresses or []
+                rfp.member_email_addresses.append(member["email"])
+
+            yield rfp.dict()

{airbyte_source_google_drive-0.2.0.dev202501272304 → airbyte_source_google_drive-0.2.0.dev202502031727}/source_google_drive/utils.py

@@ -15,6 +15,8 @@ def get_folder_id(url_string: str) -> str:
         if parsed_url.scheme != "https" or parsed_url.netloc != "drive.google.com":
             raise ValueError("Folder URL has to be of the form https://drive.google.com/drive/folders/<folder_id>")
         path_segments = list(filter(None, parsed_url.path.split("/")))
+        if path_segments[-1] in ["my-drive", "home"]:
+            return "root"
         if path_segments[-2] != "folders" or len(path_segments) < 3:
             raise ValueError("Folder URL has to be of the form https://drive.google.com/drive/folders/<folder_id>")
         return path_segments[-1]

airbyte_source_google_drive-0.2.0.dev202501272304/source_google_drive/temp_mock.py

@@ -1,36 +0,0 @@
-# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
-
-from typing import Any, Iterator
-
-
-def _get_looping_google_api_list_response(service: Any, key: str, args: dict[str, Any]) -> Iterator[dict[str, Any]]:
-    domain = args.get("domain") or args.get("groupKey")
-
-    if key == "users":
-        # Mocking 5 users
-        for i in range(1, 6):
-            yield {
-                "primaryEmail": f"user{i}@{domain}",
-                "name": {"fullName": f"User {i}"},
-                "emails": [{"address": f"user{i}@{domain}"}],
-            }
-
-    elif key == "groups":
-        # Mocking 5 groups
-        for i in range(1, 6):
-            yield {
-                "email": f"group{i}@{domain}",
-                "name": f"Group {i}",
-                "id": f"group-id-{i}",
-            }
-
-    elif key == "members":
-        # Mocking 5 members per group
-        group_id = args.get("groupKey")
-        for i in range(1, 6):
-            yield {
-                "email": f"member{i}@{domain}",
-                "role": "MEMBER",
-                "type": "USER",
-                "groupId": group_id,
-            }