airbyte-source-google-drive 0.2.0.dev202501272304__py3-none-any.whl → 0.2.0.dev202502031727__py3-none-any.whl

{airbyte_source_google_drive-0.2.0.dev202501272304.dist-info → airbyte_source_google_drive-0.2.0.dev202502031727.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: airbyte-source-google-drive
-Version: 0.2.0.dev202501272304
+Version: 0.2.0.dev202502031727
 Summary: Source implementation for Google Drive.
 License: ELv2
 Author: Airbyte
@@ -10,7 +10,7 @@ Classifier: License :: Other/Proprietary License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: airbyte-cdk[file-based] (==6.26.0.dev04105)
+Requires-Dist: airbyte-cdk[file-based] (==6.26.0.dev04106)
 Requires-Dist: google-api-python-client (==2.104.0)
 Requires-Dist: google-api-python-client-stubs (==1.18.0)
 Requires-Dist: google-auth-httplib2 (==0.1.1)

airbyte_source_google_drive-0.2.0.dev202502031727.dist-info/RECORD

@@ -0,0 +1,11 @@
+source_google_drive/__init__.py,sha256=SAgPIoGpjdHTY7mvf__A99FiElBh3WvdKSHLstg-BUE,134
+source_google_drive/exceptions.py,sha256=6sBxbdZhj0CARkctfBaNgGHB_GGcFUkY4cnMUWyOruM,268
+source_google_drive/run.py,sha256=5AQd906FbSGaNZj0udrr-VKmiSS4nsor7F7noMjsdi0,677
+source_google_drive/source.py,sha256=OyKTjPg80KR8yZiYiC3YSi6E6egothtAn3YAHAS6Q6A,2839
+source_google_drive/spec.py,sha256=YPOmOInQfmwHeB6Ay2z78LNvHsggncwU1bchQOsAZSk,4268
+source_google_drive/stream_reader.py,sha256=h4p3aafE8U9BqYgRBGXuADEhm4NCGMqYl2ExnQkj6bU,20266
+source_google_drive/utils.py,sha256=ewR-kBKLmtD-s7zqCfGECfzWYF43tpQdscAQIlUEkR8,1022
+airbyte_source_google_drive-0.2.0.dev202502031727.dist-info/METADATA,sha256=rMydjQTVW8_21SbChXDmp3i-LbEYiZi1OwOPvEKHh_k,5536
+airbyte_source_google_drive-0.2.0.dev202502031727.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
+airbyte_source_google_drive-0.2.0.dev202502031727.dist-info/entry_points.txt,sha256=YgpJf0nA5Mn0B7YC9VOFI847vz1jI6U4q7BeLUOXa54,67
+airbyte_source_google_drive-0.2.0.dev202502031727.dist-info/RECORD,,

source_google_drive/stream_reader.py

@@ -27,9 +27,6 @@ from source_google_drive.utils import get_folder_id
 from .exceptions import ErrorDownloadingFile, ErrorFetchingMetadata
 from .spec import SourceGoogleDriveSpec
 
-# remove this, just using while test credentials are prepared
-from .temp_mock import _get_looping_google_api_list_response
-
 
 FOLDER_MIME_TYPE = "application/vnd.google-apps.folder"
 GOOGLE_DOC_MIME_TYPE = "application/vnd.google-apps.document"
@@ -140,76 +137,6 @@ class SourceGoogleDriveStreamReader(AbstractFileBasedStreamReader):
 
         return self._drive_service
 
-    def _get_looping_google_api_list_response(
-        self, service: Any, key: str, args: dict[str, Any], logger: logging.Logger
-    ) -> Iterator[dict[str, Any]]:
-        try:
-            looping = True
-            next_page_token: str | None = None
-            while looping:
-                rsp = service.list(pageToken=next_page_token, **args).execute()
-                next_page_token = rsp.get("nextPageToken")
-                items: list[dict[str, Any]] = rsp.get(key)
-
-                if items is None or len(items) == 0:
-                    looping = False
-                    break
-
-                if rsp.get("nextPageToken") is None:
-                    looping = False
-                else:
-                    next_page_token = rsp.get("nextPageToken")
-
-                for item in items:
-                    yield item
-        except Exception as e:
-            logger.info(f"There was an error listing {key} with {args}: {str(e)}")
-            logger.info(f"Backing off to mocked data for development purposes")
-            yield from _get_looping_google_api_list_response(service, key, args)
-
-    def load_identity_groups(self, logger: logging.Logger) -> Dict[str, Any]:
-        domain = self.config.delivery_method.domain
-        if not domain:
-            raise Exception("No domain was provided")
-        if self.google_drive_service is None or self.google_drive_service._http.credentials is None:
-            raise Exception("No auth found")
-
-        directory_service = build("admin", "directory_v1", credentials=self.google_drive_service._http.credentials)
-        users_api = directory_service.users()
-        groups_api = directory_service.groups()
-        members_api = directory_service.members()
-
-        # here is failing
-        for user in self._get_looping_google_api_list_response(users_api, "users", {"domain": domain}, logger):
-            rfp = RemoteFileIdentity(
-                id=uuid.uuid4(),
-                remote_id=user["primaryEmail"],
-                name=user["name"]["fullName"] if user["name"] is not None else None,
-                email_address=user["primaryEmail"],
-                member_email_addresses=[x["address"] for x in user["emails"]],
-                type=RemoteFileIdentityType.USER,
-                modified_at=datetime_now(),
-            )
-            yield rfp.dict()
-
-        for group in self._get_looping_google_api_list_response(groups_api, "groups", {"domain": domain}, logger):
-            rfp = RemoteFileIdentity(
-                id=uuid.uuid4(),
-                remote_id=group["email"],
-                name=group["name"],
-                email_address=group["email"],
-                type=RemoteFileIdentityType.GROUP,
-                modified_at=datetime_now(),
-            )
-
-            for member in self._get_looping_google_api_list_response(members_api, "members", {"groupKey": group["id"]}, logger):
-                rfp.member_email_addresses = rfp.member_email_addresses or []
-                rfp.member_email_addresses.append(member["email"])
-
-            yield rfp.dict()
-
-        return ""
-
     def get_matching_files(self, globs: List[str], prefix: Optional[str], logger: logging.Logger) -> Iterable[RemoteFile]:
         """
         Get all files matching the specified glob patterns.
@@ -443,3 +370,69 @@ class SourceGoogleDriveStreamReader(AbstractFileBasedStreamReader):
             allowed_identity_remote_ids=[p.remote_id for p in remote_identities],
             publicly_accessible=is_public,
         ).dict(exclude_none=True)
+
+    def _get_looping_google_api_list_response(
+        self, service: Any, key: str, args: dict[str, Any], logger: logging.Logger
+    ) -> Iterator[dict[str, Any]]:
+        try:
+            looping = True
+            next_page_token: str | None = None
+            while looping:
+                rsp = service.list(pageToken=next_page_token, **args).execute()
+                next_page_token = rsp.get("nextPageToken")
+                items: list[dict[str, Any]] = rsp.get(key)
+
+                if items is None or len(items) == 0:
+                    looping = False
+                    break
+
+                if rsp.get("nextPageToken") is None:
+                    looping = False
+                else:
+                    next_page_token = rsp.get("nextPageToken")
+
+                for item in items:
+                    yield item
+        except Exception as e:
+            logger.error(f"There was an error listing {key} with {args}: {str(e)}")
+            raise e
+
+    def load_identity_groups(self, logger: logging.Logger) -> Dict[str, Any]:
+        domain = self.config.delivery_method.domain
+        if not domain:
+            raise Exception("No domain was provided")
+        if self.google_drive_service is None or self.google_drive_service._http.credentials is None:
+            raise Exception("No auth found")
+
+        directory_service = build("admin", "directory_v1", credentials=self.google_drive_service._http.credentials)
+        users_api = directory_service.users()
+        groups_api = directory_service.groups()
+        members_api = directory_service.members()
+
+        for user in self._get_looping_google_api_list_response(users_api, "users", {"domain": domain}, logger):
+            rfp = RemoteFileIdentity(
+                id=uuid.uuid4(),
+                remote_id=user["primaryEmail"],
+                name=user["name"]["fullName"] if user["name"] is not None else None,
+                email_address=user["primaryEmail"],
+                member_email_addresses=[x["address"] for x in user["emails"]],
+                type=RemoteFileIdentityType.USER,
+                modified_at=datetime_now(),
+            )
+            yield rfp.dict()
+
+        for group in self._get_looping_google_api_list_response(groups_api, "groups", {"domain": domain}, logger):
+            rfp = RemoteFileIdentity(
+                id=uuid.uuid4(),
+                remote_id=group["email"],
+                name=group["name"],
+                email_address=group["email"],
+                type=RemoteFileIdentityType.GROUP,
+                modified_at=datetime_now(),
+            )
+
+            for member in self._get_looping_google_api_list_response(members_api, "members", {"groupKey": group["id"]}, logger):
+                rfp.member_email_addresses = rfp.member_email_addresses or []
+                rfp.member_email_addresses.append(member["email"])
+
+            yield rfp.dict()

source_google_drive/utils.py

@@ -15,6 +15,8 @@ def get_folder_id(url_string: str) -> str:
         if parsed_url.scheme != "https" or parsed_url.netloc != "drive.google.com":
             raise ValueError("Folder URL has to be of the form https://drive.google.com/drive/folders/<folder_id>")
         path_segments = list(filter(None, parsed_url.path.split("/")))
+        if path_segments[-1] in ["my-drive", "home"]:
+            return "root"
         if path_segments[-2] != "folders" or len(path_segments) < 3:
             raise ValueError("Folder URL has to be of the form https://drive.google.com/drive/folders/<folder_id>")
         return path_segments[-1]

airbyte_source_google_drive-0.2.0.dev202501272304.dist-info/RECORD

@@ -1,12 +0,0 @@
-source_google_drive/__init__.py,sha256=SAgPIoGpjdHTY7mvf__A99FiElBh3WvdKSHLstg-BUE,134
-source_google_drive/exceptions.py,sha256=6sBxbdZhj0CARkctfBaNgGHB_GGcFUkY4cnMUWyOruM,268
-source_google_drive/run.py,sha256=5AQd906FbSGaNZj0udrr-VKmiSS4nsor7F7noMjsdi0,677
-source_google_drive/source.py,sha256=OyKTjPg80KR8yZiYiC3YSi6E6egothtAn3YAHAS6Q6A,2839
-source_google_drive/spec.py,sha256=YPOmOInQfmwHeB6Ay2z78LNvHsggncwU1bchQOsAZSk,4268
-source_google_drive/stream_reader.py,sha256=6dQOzlsMB2XoMIAdJH97Rb_dKVyDQaWeepXAN0GxxzA,20575
-source_google_drive/temp_mock.py,sha256=NdvsAltcZBIqnCBOQuU9R3_yEV5v-bHBR9T_BdjPtU4,1109
-source_google_drive/utils.py,sha256=1Fe3J4KXI1iIf4rklgLoR4p1xkCzLQI7zj5dkyi4Vt0,942
-airbyte_source_google_drive-0.2.0.dev202501272304.dist-info/METADATA,sha256=bSvWdrIpQ1GRLS1JDsssYJy2aP0BuaZMe8IfEtyDNP8,5536
-airbyte_source_google_drive-0.2.0.dev202501272304.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
-airbyte_source_google_drive-0.2.0.dev202501272304.dist-info/entry_points.txt,sha256=YgpJf0nA5Mn0B7YC9VOFI847vz1jI6U4q7BeLUOXa54,67
-airbyte_source_google_drive-0.2.0.dev202501272304.dist-info/RECORD,,

source_google_drive/temp_mock.py

@@ -1,36 +0,0 @@
-# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
-
-from typing import Any, Iterator
-
-
-def _get_looping_google_api_list_response(service: Any, key: str, args: dict[str, Any]) -> Iterator[dict[str, Any]]:
-    domain = args.get("domain") or args.get("groupKey")
-
-    if key == "users":
-        # Mocking 5 users
-        for i in range(1, 6):
-            yield {
-                "primaryEmail": f"user{i}@{domain}",
-                "name": {"fullName": f"User {i}"},
-                "emails": [{"address": f"user{i}@{domain}"}],
-            }
-
-    elif key == "groups":
-        # Mocking 5 groups
-        for i in range(1, 6):
-            yield {
-                "email": f"group{i}@{domain}",
-                "name": f"Group {i}",
-                "id": f"group-id-{i}",
-            }
-
-    elif key == "members":
-        # Mocking 5 members per group
-        group_id = args.get("groupKey")
-        for i in range(1, 6):
-            yield {
-                "email": f"member{i}@{domain}",
-                "role": "MEMBER",
-                "type": "USER",
-                "groupId": group_id,
-            }