airbyte-source-google-drive 0.3.3__py3-none-any.whl → 0.4.0.dev202504091818__py3-none-any.whl

{airbyte_source_google_drive-0.3.3.dist-info → airbyte_source_google_drive-0.4.0.dev202504091818.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: airbyte-source-google-drive
-Version: 0.3.3
+Version: 0.4.0.dev202504091818
 Summary: Source implementation for Google Drive.
 License: ELv2
 Author: Airbyte
@@ -10,7 +10,7 @@ Classifier: License :: Other/Proprietary License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: airbyte-cdk[file-based] (>=6.38.5,<7.0.0)
+Requires-Dist: airbyte-cdk[file-based] (==6.45.0.dev04107)
 Requires-Dist: google-api-python-client (==2.104.0)
 Requires-Dist: google-api-python-client-stubs (==1.18.0)
 Requires-Dist: google-auth-httplib2 (==0.1.1)

{airbyte_source_google_drive-0.3.3.dist-info → airbyte_source_google_drive-0.4.0.dev202504091818.dist-info}/RECORD

@@ -6,9 +6,9 @@ source_google_drive/schemas/identities.json,sha256=JXBR_v0wpfDKiWVzLoc8bPs33x5CG
 source_google_drive/source.py,sha256=Hmz60uMCQZyA9AhCN0bOUSW2iL3GW6VTwI5vrK0D1RY,4140
 source_google_drive/spec.py,sha256=-WkA2zGuQtf3G7uK8uq9BnimUlQh0s3vsqROmIHOgzI,4718
 source_google_drive/stream_permissions_reader.py,sha256=88bSJBC5n2svioTDwgVqf0x5yXRxityk_i0xokWEJcQ,9026
-source_google_drive/stream_reader.py,sha256=d1c-u3iGI5i7RQ3rTCoDV7JXHwyuKIMp0d1GMQgOsfA,14246
+source_google_drive/stream_reader.py,sha256=r7eSUJ1CTgaZsMg3TljUDjSNidTyAyfRGfv2vxhLdlQ,14909
 source_google_drive/utils.py,sha256=ewR-kBKLmtD-s7zqCfGECfzWYF43tpQdscAQIlUEkR8,1022
-airbyte_source_google_drive-0.3.3.dist-info/METADATA,sha256=qLevpcn6boj2tUWKCHvRqYyFMSlvOiWBxztIF7xDONU,5518
-airbyte_source_google_drive-0.3.3.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
-airbyte_source_google_drive-0.3.3.dist-info/entry_points.txt,sha256=YgpJf0nA5Mn0B7YC9VOFI847vz1jI6U4q7BeLUOXa54,67
-airbyte_source_google_drive-0.3.3.dist-info/RECORD,,
+airbyte_source_google_drive-0.4.0.dev202504091818.dist-info/METADATA,sha256=3kVi44z3GI6xpjSQPUhi44MAUSk6FoldVXQlYvhoK5w,5536
+airbyte_source_google_drive-0.4.0.dev202504091818.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
+airbyte_source_google_drive-0.4.0.dev202504091818.dist-info/entry_points.txt,sha256=YgpJf0nA5Mn0B7YC9VOFI847vz1jI6U4q7BeLUOXa54,67
+airbyte_source_google_drive-0.4.0.dev202504091818.dist-info/RECORD,,

source_google_drive/stream_reader.py

@@ -6,27 +6,26 @@
 import io
 import json
 import logging
-import uuid
+import os
 from datetime import datetime
 from io import IOBase
 from os.path import getsize
-from typing import Any, Dict, Iterable, Iterator, List, Optional, Set, Tuple
+from typing import Dict, Iterable, List, Optional, Set, Tuple
 
-import pytz
 from google.oauth2 import credentials, service_account
 from googleapiclient.discovery import build
 from googleapiclient.http import MediaIoBaseDownload
 
 from airbyte_cdk import AirbyteTracedException, FailureType
+from airbyte_cdk.models import AirbyteRecordMessageFileReference
 from airbyte_cdk.sources.file_based.exceptions import FileSizeLimitError
 from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode
+from airbyte_cdk.sources.file_based.file_record_data import FileRecordData
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
-from airbyte_cdk.sources.streams.core import package_name_from_class
-from airbyte_cdk.sources.utils.schema_helpers import InternalConfig, ResourceSchemaLoader
 from source_google_drive.utils import get_folder_id
 
 from .exceptions import ErrorDownloadingFile, ErrorFetchingMetadata
-from .spec import RemoteIdentity, RemoteIdentityType, RemotePermissions, SourceGoogleDriveSpec
+from .spec import SourceGoogleDriveSpec
 
 
 FOLDER_MIME_TYPE = "application/vnd.google-apps.folder"
@@ -245,7 +244,9 @@ class SourceGoogleDriveStreamReader(AbstractFileBasedStreamReader):
         except Exception as e:
             raise ErrorFetchingMetadata(f"An error occurred while retrieving file size: {str(e)}")
 
-    def get_file(self, file: GoogleDriveRemoteFile, local_directory: str, logger: logging.Logger) -> Dict[str, str | int]:
+    def upload(
+        self, file: GoogleDriveRemoteFile, local_directory: str, logger: logging.Logger
+    ) -> Tuple[FileRecordData, AirbyteRecordMessageFileReference]:
         """
         Downloads a file from Google Drive to a specified local directory.
 
@@ -265,15 +266,18 @@ class SourceGoogleDriveStreamReader(AbstractFileBasedStreamReader):
             raise FileSizeLimitError(message=message, internal_message=message, failure_type=FailureType.config_error)
 
         try:
-            file_relative_path, local_file_path, absolute_file_path = self._get_file_transfer_paths(file, local_directory)
+            file_paths = self._get_file_transfer_paths(file, local_directory)
+            local_file_path = file_paths[self.LOCAL_FILE_PATH]
+            file_relative_path = file_paths[self.FILE_RELATIVE_PATH]
+            file_name = file_paths[self.FILE_NAME]
 
             if self._is_exportable_document(file.original_mime_type):
                 request = self.google_drive_service.files().export_media(fileId=file.id, mimeType=file.mime_type)
 
                 file_extension = DOWNLOADABLE_DOCUMENTS_MIME_TYPES[file.original_mime_type][DOCUMENT_FILE_EXTENSION_KEY]
                 local_file_path += file_extension
-                absolute_file_path += file_extension
                 file_relative_path += file_extension
+                file_name += file_extension
             else:
                 request = self.google_drive_service.files().get_media(fileId=file.id)
 
@@ -285,10 +289,24 @@ class SourceGoogleDriveStreamReader(AbstractFileBasedStreamReader):
                     progress = status.resumable_progress / status.total_size * 100 if status.total_size else 0
                     logger.info(f"Processing file {file.uri}, progress: {progress:.2f}%")
 
+            logger.info(f"Finished uploading file {file.uri} to {local_file_path}")
             # native google objects seems to be reporting lower size through the api than the final download size
             file_size = getsize(local_file_path)
 
-            return {"file_url": absolute_file_path, "bytes": file_size, "file_relative_path": file_relative_path}
+            file_record_data = FileRecordData(
+                folder=file_paths[self.FILE_FOLDER],
+                filename=file_name,
+                bytes=file_size,
+                id=file.id,
+                mime_type=file.mime_type,
+                updated_at=file.last_modified.strftime("%Y-%m-%dT%H:%M:%S.%fZ"),
+            )
+            file_reference = AirbyteRecordMessageFileReference(
+                staging_file_url=local_file_path,
+                source_file_relative_path=file_relative_path,
+                file_size_bytes=file_size,
+            )
+            return file_record_data, file_reference
 
         except Exception as e:
             raise ErrorDownloadingFile(f"There was an error while trying to download the file {file.uri}: {str(e)}")