unstructured-ingest 0.6.2__py3-none-any.whl → 0.7.0__py3-none-any.whl

unstructured_ingest/{v2/processes → processes}/connectors/elasticsearch/__init__.py

@@ -1,4 +1,4 @@
-from unstructured_ingest.v2.processes.connector_registry import (
+from unstructured_ingest.processes.connector_registry import (
     add_destination_entry,
     add_source_entry,
 )

unstructured_ingest/{v2/processes → processes}/connectors/elasticsearch/elasticsearch.py

@@ -8,20 +8,20 @@ from typing import TYPE_CHECKING, Any, Generator, Optional, Union
 
 from pydantic import BaseModel, Field, Secret, SecretStr, field_validator
 
+from unstructured_ingest.data_types.file_data import (
+    BatchFileData,
+    BatchItem,
+    FileData,
+    FileDataSourceMetadata,
+    SourceIdentifiers,
+)
 from unstructured_ingest.error import (
     DestinationConnectionError,
     SourceConnectionError,
     SourceConnectionNetworkError,
     WriteError,
 )
-from unstructured_ingest.utils.data_prep import (
-    batch_generator,
-    flatten_dict,
-    generator_batching_wbytes,
-)
-from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.constants import RECORD_ID_LABEL
-from unstructured_ingest.v2.interfaces import (
+from unstructured_ingest.interfaces import (
     AccessConfig,
     ConnectionConfig,
     Downloader,
@@ -35,19 +35,19 @@ from unstructured_ingest.v2.interfaces import (
     UploadStagerConfig,
     download_responses,
 )
-from unstructured_ingest.v2.logger import logger
-from unstructured_ingest.v2.processes.connector_registry import (
+from unstructured_ingest.logger import logger
+from unstructured_ingest.processes.connector_registry import (
     DestinationRegistryEntry,
     SourceRegistryEntry,
 )
-from unstructured_ingest.v2.types.file_data import (
-    BatchFileData,
-    BatchItem,
-    FileData,
-    FileDataSourceMetadata,
-    SourceIdentifiers,
+from unstructured_ingest.utils.constants import RECORD_ID_LABEL
+from unstructured_ingest.utils.data_prep import (
+    batch_generator,
+    flatten_dict,
+    generator_batching_wbytes,
+    get_enhanced_element_id,
 )
-from unstructured_ingest.v2.utils import get_enhanced_element_id
+from unstructured_ingest.utils.dep_check import requires_dependencies
 
 if TYPE_CHECKING:
     from elasticsearch import Elasticsearch as ElasticsearchClient

unstructured_ingest/{v2/processes → processes}/connectors/elasticsearch/opensearch.py

@@ -7,17 +7,16 @@ from pydantic import BaseModel, Field, Secret, field_validator
 from unstructured_ingest.error import (
     DestinationConnectionError,
 )
-from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.interfaces import (
+from unstructured_ingest.interfaces import (
     AccessConfig,
     ConnectionConfig,
 )
-from unstructured_ingest.v2.logger import logger
-from unstructured_ingest.v2.processes.connector_registry import (
+from unstructured_ingest.logger import logger
+from unstructured_ingest.processes.connector_registry import (
     DestinationRegistryEntry,
     SourceRegistryEntry,
 )
-from unstructured_ingest.v2.processes.connectors.elasticsearch.elasticsearch import (
+from unstructured_ingest.processes.connectors.elasticsearch.elasticsearch import (
     ElasticsearchDownloader,
     ElasticsearchDownloaderConfig,
     ElasticsearchIndexer,
@@ -27,6 +26,7 @@ from unstructured_ingest.v2.processes.connectors.elasticsearch.elasticsearch imp
     ElasticsearchUploadStager,
     ElasticsearchUploadStagerConfig,
 )
+from unstructured_ingest.utils.dep_check import requires_dependencies
 
 if TYPE_CHECKING:
     from opensearchpy import OpenSearch

unstructured_ingest/{v2/processes → processes}/connectors/fsspec/__init__.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from unstructured_ingest.v2.processes.connector_registry import (
+from unstructured_ingest.processes.connector_registry import (
     add_destination_entry,
     add_source_entry,
 )

unstructured_ingest/{v2/processes → processes}/connectors/fsspec/azure.py

@@ -7,14 +7,14 @@ from typing import TYPE_CHECKING, Any, Generator, Optional
 
 from pydantic import Field, Secret
 
-from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.errors import ProviderError, UserAuthError, UserError
-from unstructured_ingest.v2.logger import logger
-from unstructured_ingest.v2.processes.connector_registry import (
+from unstructured_ingest.data_types.file_data import FileDataSourceMetadata
+from unstructured_ingest.errors_v2 import ProviderError, UserAuthError, UserError
+from unstructured_ingest.logger import logger
+from unstructured_ingest.processes.connector_registry import (
     DestinationRegistryEntry,
     SourceRegistryEntry,
 )
-from unstructured_ingest.v2.processes.connectors.fsspec.fsspec import (
+from unstructured_ingest.processes.connectors.fsspec.fsspec import (
     FsspecAccessConfig,
     FsspecConnectionConfig,
     FsspecDownloader,
@@ -24,12 +24,12 @@ from unstructured_ingest.v2.processes.connectors.fsspec.fsspec import (
     FsspecUploader,
     FsspecUploaderConfig,
 )
-from unstructured_ingest.v2.processes.connectors.fsspec.utils import json_serial, sterilize_dict
-from unstructured_ingest.v2.processes.utils.blob_storage import (
+from unstructured_ingest.processes.connectors.fsspec.utils import json_serial, sterilize_dict
+from unstructured_ingest.processes.utils.blob_storage import (
     BlobStoreUploadStager,
     BlobStoreUploadStagerConfig,
 )
-from unstructured_ingest.v2.types.file_data import FileDataSourceMetadata
+from unstructured_ingest.utils.dep_check import requires_dependencies
 
 if TYPE_CHECKING:
     from adlfs import AzureBlobFileSystem

unstructured_ingest/{v2/processes → processes}/connectors/fsspec/box.py

@@ -9,14 +9,14 @@ from dateutil import parser
 from pydantic import Field, Secret
 from pydantic.functional_validators import BeforeValidator
 
-from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.errors import ProviderError, UserAuthError, UserError
-from unstructured_ingest.v2.logger import logger
-from unstructured_ingest.v2.processes.connector_registry import (
+from unstructured_ingest.data_types.file_data import FileDataSourceMetadata
+from unstructured_ingest.errors_v2 import ProviderError, UserAuthError, UserError
+from unstructured_ingest.logger import logger
+from unstructured_ingest.processes.connector_registry import (
     DestinationRegistryEntry,
     SourceRegistryEntry,
 )
-from unstructured_ingest.v2.processes.connectors.fsspec.fsspec import (
+from unstructured_ingest.processes.connectors.fsspec.fsspec import (
     FsspecAccessConfig,
     FsspecConnectionConfig,
     FsspecDownloader,
@@ -26,12 +26,12 @@ from unstructured_ingest.v2.processes.connectors.fsspec.fsspec import (
     FsspecUploader,
     FsspecUploaderConfig,
 )
-from unstructured_ingest.v2.processes.connectors.utils import conform_string_to_dict
-from unstructured_ingest.v2.processes.utils.blob_storage import (
+from unstructured_ingest.processes.connectors.utils import conform_string_to_dict
+from unstructured_ingest.processes.utils.blob_storage import (
     BlobStoreUploadStager,
     BlobStoreUploadStagerConfig,
 )
-from unstructured_ingest.v2.types.file_data import FileDataSourceMetadata
+from unstructured_ingest.utils.dep_check import requires_dependencies
 
 if TYPE_CHECKING:
     from boxfs import BoxFileSystem

unstructured_ingest/{v2/processes → processes}/connectors/fsspec/dropbox.py

@@ -6,21 +6,21 @@ from typing import TYPE_CHECKING, Any, Optional
 
 from pydantic import Field, Secret
 
-from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.errors import (
+from unstructured_ingest.data_types.file_data import FileDataSourceMetadata
+from unstructured_ingest.errors_v2 import (
     ProviderError,
     UserAuthError,
     UserError,
 )
-from unstructured_ingest.v2.errors import (
+from unstructured_ingest.errors_v2 import (
     RateLimitError as CustomRateLimitError,
 )
-from unstructured_ingest.v2.logger import logger
-from unstructured_ingest.v2.processes.connector_registry import (
+from unstructured_ingest.logger import logger
+from unstructured_ingest.processes.connector_registry import (
     DestinationRegistryEntry,
     SourceRegistryEntry,
 )
-from unstructured_ingest.v2.processes.connectors.fsspec.fsspec import (
+from unstructured_ingest.processes.connectors.fsspec.fsspec import (
     FsspecAccessConfig,
     FsspecConnectionConfig,
     FsspecDownloader,
@@ -30,11 +30,11 @@ from unstructured_ingest.v2.processes.connectors.fsspec.fsspec import (
     FsspecUploader,
     FsspecUploaderConfig,
 )
-from unstructured_ingest.v2.processes.utils.blob_storage import (
+from unstructured_ingest.processes.utils.blob_storage import (
     BlobStoreUploadStager,
     BlobStoreUploadStagerConfig,
 )
-from unstructured_ingest.v2.types.file_data import FileDataSourceMetadata
+from unstructured_ingest.utils.dep_check import requires_dependencies
 
 if TYPE_CHECKING:
     pass

unstructured_ingest/{v2/processes → processes}/connectors/fsspec/fsspec.py

@@ -12,7 +12,12 @@ from uuid import NAMESPACE_DNS, uuid5
 
 from pydantic import BaseModel, Field, Secret
 
-from unstructured_ingest.v2.interfaces import (
+from unstructured_ingest.data_types.file_data import (
+    FileData,
+    FileDataSourceMetadata,
+    SourceIdentifiers,
+)
+from unstructured_ingest.interfaces import (
     AccessConfig,
     ConnectionConfig,
     Downloader,
@@ -23,13 +28,8 @@ from unstructured_ingest.v2.interfaces import (
     Uploader,
     UploaderConfig,
 )
-from unstructured_ingest.v2.logger import logger
-from unstructured_ingest.v2.processes.connectors.fsspec.utils import sterilize_dict
-from unstructured_ingest.v2.types.file_data import (
-    FileData,
-    FileDataSourceMetadata,
-    SourceIdentifiers,
-)
+from unstructured_ingest.logger import logger
+from unstructured_ingest.processes.connectors.fsspec.utils import sterilize_dict
 
 if TYPE_CHECKING:
     from fsspec import AbstractFileSystem

unstructured_ingest/{v2/processes → processes}/connectors/fsspec/gcs.py

@@ -9,15 +9,14 @@ from typing import TYPE_CHECKING, Any, Generator, Optional, Union
 from dateutil import parser
 from pydantic import Field, Secret
 
-from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.utils.string_and_date_utils import json_to_dict
-from unstructured_ingest.v2.errors import ProviderError, UserError
-from unstructured_ingest.v2.logger import logger
-from unstructured_ingest.v2.processes.connector_registry import (
+from unstructured_ingest.data_types.file_data import FileDataSourceMetadata
+from unstructured_ingest.errors_v2 import ProviderError, UserError
+from unstructured_ingest.logger import logger
+from unstructured_ingest.processes.connector_registry import (
     DestinationRegistryEntry,
     SourceRegistryEntry,
 )
-from unstructured_ingest.v2.processes.connectors.fsspec.fsspec import (
+from unstructured_ingest.processes.connectors.fsspec.fsspec import (
     FsspecAccessConfig,
     FsspecConnectionConfig,
     FsspecDownloader,
@@ -27,11 +26,12 @@ from unstructured_ingest.v2.processes.connectors.fsspec.fsspec import (
     FsspecUploader,
     FsspecUploaderConfig,
 )
-from unstructured_ingest.v2.processes.utils.blob_storage import (
+from unstructured_ingest.processes.utils.blob_storage import (
     BlobStoreUploadStager,
     BlobStoreUploadStagerConfig,
 )
-from unstructured_ingest.v2.types.file_data import FileDataSourceMetadata
+from unstructured_ingest.utils.dep_check import requires_dependencies
+from unstructured_ingest.utils.string_and_date_utils import json_to_dict
 
 if TYPE_CHECKING:
     from gcsfs import GCSFileSystem

unstructured_ingest/{v2/processes → processes}/connectors/fsspec/s3.py

@@ -6,14 +6,16 @@ from typing import TYPE_CHECKING, Any, Generator, Optional
 
 from pydantic import Field, Secret
 
-from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.errors import ProviderError, UserAuthError, UserError
-from unstructured_ingest.v2.logger import logger
-from unstructured_ingest.v2.processes.connector_registry import (
+from unstructured_ingest.data_types.file_data import (
+    FileDataSourceMetadata,
+)
+from unstructured_ingest.errors_v2 import ProviderError, UserAuthError, UserError
+from unstructured_ingest.logger import logger
+from unstructured_ingest.processes.connector_registry import (
     DestinationRegistryEntry,
     SourceRegistryEntry,
 )
-from unstructured_ingest.v2.processes.connectors.fsspec.fsspec import (
+from unstructured_ingest.processes.connectors.fsspec.fsspec import (
     FsspecAccessConfig,
     FsspecConnectionConfig,
     FsspecDownloader,
@@ -23,13 +25,11 @@ from unstructured_ingest.v2.processes.connectors.fsspec.fsspec import (
     FsspecUploader,
     FsspecUploaderConfig,
 )
-from unstructured_ingest.v2.processes.utils.blob_storage import (
+from unstructured_ingest.processes.utils.blob_storage import (
     BlobStoreUploadStager,
     BlobStoreUploadStagerConfig,
 )
-from unstructured_ingest.v2.types.file_data import (
-    FileDataSourceMetadata,
-)
+from unstructured_ingest.utils.dep_check import requires_dependencies
 
 CONNECTOR_TYPE = "s3"
 

unstructured_ingest/{v2/processes → processes}/connectors/fsspec/sftp.py

@@ -10,12 +10,12 @@ from urllib.parse import urlparse
 
 from pydantic import Field, Secret
 
-from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.processes.connector_registry import (
+from unstructured_ingest.data_types.file_data import FileData, FileDataSourceMetadata
+from unstructured_ingest.processes.connector_registry import (
     DestinationRegistryEntry,
     SourceRegistryEntry,
 )
-from unstructured_ingest.v2.processes.connectors.fsspec.fsspec import (
+from unstructured_ingest.processes.connectors.fsspec.fsspec import (
     FsspecAccessConfig,
     FsspecConnectionConfig,
     FsspecDownloader,
@@ -25,11 +25,11 @@ from unstructured_ingest.v2.processes.connectors.fsspec.fsspec import (
     FsspecUploader,
     FsspecUploaderConfig,
 )
-from unstructured_ingest.v2.processes.utils.blob_storage import (
+from unstructured_ingest.processes.utils.blob_storage import (
     BlobStoreUploadStager,
     BlobStoreUploadStagerConfig,
 )
-from unstructured_ingest.v2.types.file_data import FileData, FileDataSourceMetadata
+from unstructured_ingest.utils.dep_check import requires_dependencies
 
 if TYPE_CHECKING:
     from fsspec.implementations.sftp import SFTPFileSystem

unstructured_ingest/processes/connectors/github.py

@@ -0,0 +1,221 @@
+from dataclasses import dataclass, field
+from pathlib import Path
+from time import time
+from typing import TYPE_CHECKING, Any, Generator, Optional
+from urllib.parse import urlparse
+from uuid import NAMESPACE_DNS, uuid5
+
+from pydantic import Field, Secret, field_validator
+
+from unstructured_ingest.data_types.file_data import (
+    FileData,
+    FileDataSourceMetadata,
+    SourceIdentifiers,
+)
+from unstructured_ingest.errors_v2 import ProviderError, UserAuthError, UserError
+from unstructured_ingest.interfaces import (
+    AccessConfig,
+    ConnectionConfig,
+    Downloader,
+    DownloaderConfig,
+    Indexer,
+    IndexerConfig,
+    download_responses,
+)
+from unstructured_ingest.logger import logger
+from unstructured_ingest.processes.connector_registry import (
+    SourceRegistryEntry,
+)
+from unstructured_ingest.utils.dep_check import requires_dependencies
+
+if TYPE_CHECKING:
+    from github import ContentFile, GitTreeElement, Repository
+    from github import Github as GithubClient
+    from github.GithubException import GithubException
+    from requests import HTTPError
+
+CONNECTOR_TYPE = "github"
+
+
+class GithubAccessConfig(AccessConfig):
+    access_token: str = Field(description="Github acess token")
+
+
+class GithubConnectionConfig(ConnectionConfig):
+    access_config: Secret[GithubAccessConfig]
+    url: str = Field(description="Github url or repository owner/name pair")
+
+    @field_validator("url", mode="after")
+    def conform_url(cls, value: str):
+        parsed_url = urlparse(value)
+        return parsed_url.path
+
+    def get_full_url(self):
+        return f"https://github.com/{self.url}"
+
+    @requires_dependencies(["github"], extras="github")
+    def get_client(self) -> "GithubClient":
+        from github import Github as GithubClient
+
+        return GithubClient(login_or_token=self.access_config.get_secret_value().access_token)
+
+    def get_repo(self) -> "Repository":
+        client = self.get_client()
+        return client.get_repo(self.url)
+
+    def wrap_github_exception(self, e: "GithubException") -> Exception:
+        data = e.data
+        status_code = e.status
+        message = data.get("message")
+        if status_code == 401:
+            return UserAuthError(f"Unauthorized access to Github: {message}")
+        if 400 <= status_code < 500:
+            return UserError(message)
+        if status_code > 500:
+            return ProviderError(message)
+        logger.debug(f"unhandled github error: {e}")
+        return e
+
+    def wrap_http_error(self, e: "HTTPError") -> Exception:
+        status_code = e.response.status_code
+        if status_code == 401:
+            return UserAuthError(f"Unauthorized access to Github: {e.response.text}")
+        if 400 <= status_code < 500:
+            return UserError(e.response.text)
+        if status_code > 500:
+            return ProviderError(e.response.text)
+        logger.debug(f"unhandled http error: {e}")
+        return e
+
+    @requires_dependencies(["requests"], extras="github")
+    def wrap_error(self, e: Exception) -> Exception:
+        from github.GithubException import GithubException
+        from requests import HTTPError
+
+        if isinstance(e, GithubException):
+            return self.wrap_github_exception(e=e)
+        if isinstance(e, HTTPError):
+            return self.wrap_http_error(e=e)
+        logger.debug(f"unhandled error: {e}")
+        return e
+
+
+class GithubIndexerConfig(IndexerConfig):
+    branch: Optional[str] = Field(
+        description="Branch to index, use the default if one isn't provided", default=None
+    )
+    recursive: bool = Field(
+        description="Recursively index all files in the repository", default=True
+    )
+
+
+@dataclass
+class GithubIndexer(Indexer):
+    connection_config: GithubConnectionConfig
+    index_config: GithubIndexerConfig = field(default_factory=GithubIndexerConfig)
+    connector_type: str = CONNECTOR_TYPE
+
+    def precheck(self) -> None:
+        try:
+            self.connection_config.get_repo()
+        except Exception as e:
+            raise self.connection_config.wrap_error(e=e)
+
+    def get_branch(self) -> str:
+        repo = self.connection_config.get_repo()
+        sha = self.index_config.branch or repo.default_branch
+        return sha
+
+    def list_files(self) -> list["GitTreeElement"]:
+        repo = self.connection_config.get_repo()
+        sha = self.index_config.branch or repo.default_branch
+        git_tree = repo.get_git_tree(sha, recursive=self.index_config.recursive)
+        file_elements = [
+            element for element in git_tree.tree if element.size is not None and element.size > 0
+        ]
+        return file_elements
+
+    def convert_element(self, element: "GitTreeElement") -> FileData:
+        full_path = (
+            f"{self.connection_config.get_full_url()}/blob/{self.get_branch()}/{element.path}"
+        )
+
+        return FileData(
+            identifier=str(uuid5(NAMESPACE_DNS, full_path)),
+            connector_type=self.connector_type,
+            display_name=full_path,
+            source_identifiers=SourceIdentifiers(
+                filename=Path(element.path).name,
+                fullpath=(Path(self.get_branch()) / element.path).as_posix(),
+                rel_path=element.path,
+            ),
+            metadata=FileDataSourceMetadata(
+                url=element.url,
+                version=element.etag,
+                record_locator={},
+                date_modified=str(element.last_modified_datetime.timestamp()),
+                date_processed=str(time()),
+                filesize_bytes=element.size,
+                permissions_data=[{"mode": element.mode}],
+            ),
+        )
+
+    def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
+        for element in self.list_files():
+            yield self.convert_element(element=element)
+
+
+class GithubDownloaderConfig(DownloaderConfig):
+    pass
+
+
+@dataclass
+class GithubDownloader(Downloader):
+    download_config: GithubDownloaderConfig
+    connection_config: GithubConnectionConfig
+    connector_type: str = CONNECTOR_TYPE
+
+    @requires_dependencies(["github"], extras="github")
+    def get_file(self, file_data: FileData) -> "ContentFile":
+        from github.GithubException import UnknownObjectException
+
+        path = file_data.source_identifiers.relative_path
+        repo = self.connection_config.get_repo()
+
+        try:
+            content_file = repo.get_contents(path)
+        except UnknownObjectException as e:
+            logger.error(f"File doesn't exists {self.connection_config.url}/{path}: {e}")
+            raise UserError(f"File not found: {path}")
+        return content_file
+
+    @requires_dependencies(["requests"], extras="github")
+    def get_contents(self, content_file: "ContentFile") -> bytes:
+        import requests
+
+        if content_file.decoded_content:
+            return content_file.decoded_content
+        download_url = content_file.download_url
+        resp = requests.get(download_url)
+        try:
+            resp.raise_for_status()
+        except requests.HTTPError as e:
+            raise self.connection_config.wrap_error(e=e)
+        return resp.content
+
+    def run(self, file_data: FileData, **kwargs: Any) -> download_responses:
+        content_file = self.get_file(file_data)
+        contents = self.get_contents(content_file)
+        download_path = self.get_download_path(file_data)
+        with download_path.open("wb") as f:
+            f.write(contents)
+        return self.generate_download_response(file_data=file_data, download_path=download_path)
+
+
+github_source_entry = SourceRegistryEntry(
+    indexer=GithubIndexer,
+    indexer_config=GithubIndexerConfig,
+    downloader=GithubDownloader,
+    downloader_config=GithubDownloaderConfig,
+    connection_config=GithubConnectionConfig,
+)

unstructured_ingest/{v2/processes → processes}/connectors/gitlab.py

@@ -8,9 +8,13 @@ from urllib.parse import urlparse
 
 from pydantic import Field, Secret, model_validator
 
+from unstructured_ingest.data_types.file_data import (
+    FileData,
+    FileDataSourceMetadata,
+    SourceIdentifiers,
+)
 from unstructured_ingest.error import SourceConnectionError
-from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.interfaces import (
+from unstructured_ingest.interfaces import (
     AccessConfig,
     ConnectionConfig,
     Downloader,
@@ -19,13 +23,9 @@ from unstructured_ingest.v2.interfaces import (
     Indexer,
     IndexerConfig,
 )
-from unstructured_ingest.v2.logger import logger
-from unstructured_ingest.v2.processes.connector_registry import SourceRegistryEntry
-from unstructured_ingest.v2.types.file_data import (
-    FileData,
-    FileDataSourceMetadata,
-    SourceIdentifiers,
-)
+from unstructured_ingest.logger import logger
+from unstructured_ingest.processes.connector_registry import SourceRegistryEntry
+from unstructured_ingest.utils.dep_check import requires_dependencies
 
 CONNECTOR_TYPE = "gitlab"
 if TYPE_CHECKING:

unstructured_ingest/{v2/processes → processes}/connectors/google_drive.py

@@ -9,13 +9,16 @@ from dateutil import parser
 from pydantic import Field, Secret
 from pydantic.functional_validators import BeforeValidator
 
+from unstructured_ingest.data_types.file_data import (
+    FileData,
+    FileDataSourceMetadata,
+    SourceIdentifiers,
+)
 from unstructured_ingest.error import (
     SourceConnectionError,
     SourceConnectionNetworkError,
 )
-from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.utils.google_filetype import GOOGLE_DRIVE_EXPORT_TYPES
-from unstructured_ingest.v2.interfaces import (
+from unstructured_ingest.interfaces import (
     AccessConfig,
     ConnectionConfig,
     Downloader,
@@ -24,21 +27,27 @@ from unstructured_ingest.v2.interfaces import (
     Indexer,
     IndexerConfig,
 )
-from unstructured_ingest.v2.logger import logger
-from unstructured_ingest.v2.processes.connector_registry import SourceRegistryEntry
-from unstructured_ingest.v2.processes.connectors.utils import conform_string_to_dict
-from unstructured_ingest.v2.types.file_data import (
-    FileData,
-    FileDataSourceMetadata,
-    SourceIdentifiers,
-)
-
-CONNECTOR_TYPE = "google_drive"
+from unstructured_ingest.logger import logger
+from unstructured_ingest.processes.connector_registry import SourceRegistryEntry
+from unstructured_ingest.processes.connectors.utils import conform_string_to_dict
+from unstructured_ingest.utils.dep_check import requires_dependencies
 
 if TYPE_CHECKING:
     from googleapiclient.discovery import Resource as GoogleAPIResource
     from googleapiclient.http import MediaIoBaseDownload
 
+CONNECTOR_TYPE = "google_drive"
+
+GOOGLE_DRIVE_EXPORT_TYPES = {
+    "application/vnd.google-apps.document": "application/"
+    "vnd.openxmlformats-officedocument.wordprocessingml.document",
+    "application/vnd.google-apps.spreadsheet": "application/"
+    "vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+    "application/vnd.google-apps.presentation": "application/"
+    "vnd.openxmlformats-officedocument.presentationml.presentation",
+    "application/vnd.google-apps.photo": "image/jpeg",
+}
+
 
 class GoogleDriveAccessConfig(AccessConfig):
     service_account_key: Optional[Annotated[dict, BeforeValidator(conform_string_to_dict)]] = Field(