unstructured-ingest 1.2.32__py3-none-any.whl

unstructured_ingest/processes/connectors/salesforce.py

@@ -0,0 +1,307 @@
+"""
+Salesforce Connector
+Able to download Account, Case, Campaign, EmailMessage, Lead
+Salesforce returns everything as a list of json.
+This saves each entry as a separate file to be partitioned.
+Using JWT authorization
+https://developer.salesforce.com/docs/atlas.en-us.sfdx_dev.meta/sfdx_dev/sfdx_dev_auth_key_and_cert.htm
+https://developer.salesforce.com/docs/atlas.en-us.sfdx_dev.meta/sfdx_dev/sfdx_dev_auth_connected_app.htm
+"""
+
+import json
+from collections import OrderedDict
+from dataclasses import dataclass, field
+from email.utils import formatdate
+from pathlib import Path
+from string import Template
+from textwrap import dedent
+from typing import TYPE_CHECKING, Any, Generator, Optional, Type
+
+from dateutil import parser
+from pydantic import Field, Secret
+
+from unstructured_ingest.data_types.file_data import (
+    FileData,
+    FileDataSourceMetadata,
+    SourceIdentifiers,
+)
+from unstructured_ingest.error import (
+    MissingCategoryError,
+    SourceConnectionError,
+    SourceConnectionNetworkError,
+    ValueError,
+)
+from unstructured_ingest.interfaces import (
+    AccessConfig,
+    ConnectionConfig,
+    Downloader,
+    DownloaderConfig,
+    DownloadResponse,
+    Indexer,
+    IndexerConfig,
+)
+from unstructured_ingest.logger import logger
+from unstructured_ingest.processes.connector_registry import (
+    SourceRegistryEntry,
+)
+from unstructured_ingest.utils.dep_check import requires_dependencies
+
+CONNECTOR_TYPE = "salesforce"
+
+if TYPE_CHECKING:
+    from simple_salesforce import Salesforce
+
+SALESFORCE_API_VERSION = "57.0"
+
+# TODO: Add more categories as needed
+ACCEPTED_CATEGORIES: list[str] = ["Account", "Case", "Campaign", "EmailMessage", "Lead"]
+
+# Generic minimal email template used only
+# to process EmailMessage records as .eml files
+EMAIL_TEMPLATE = Template(
+    """MIME-Version: 1.0
+Date: $date
+Message-ID: $message_identifier
+Subject: $subject
+From: $from_email
+To: $to_email
+Content-Type: multipart/alternative; boundary="00000000000095c9b205eff92630"
+--00000000000095c9b205eff92630
+Content-Type: text/plain; charset="UTF-8"
+$textbody
+--00000000000095c9b205eff92630
+Content-Type: text/html; charset="UTF-8"
+$htmlbody
+--00000000000095c9b205eff92630--
+""",
+)
+
+
+class SalesforceAccessConfig(AccessConfig):
+    consumer_key: str
+    private_key_path: Optional[Path] = Field(
+        default=None,
+        description="Path to the private key file. Key file is usually named server.key.",
+    )
+    private_key: Optional[str] = Field(default=None, description="Contents of the private key")
+
+    def model_post_init(self, __context: Any) -> None:
+        if self.private_key_path is None and self.private_key is None:
+            raise ValueError("either private_key or private_key_path must be set")
+        if self.private_key is not None and self.private_key_path is not None:
+            raise ValueError("only one of private_key or private_key_path must be set")
+
+    @requires_dependencies(["cryptography"])
+    def get_private_key_value_and_type(self) -> tuple[str, Type]:
+        from cryptography.hazmat.primitives import serialization
+
+        if self.private_key_path and self.private_key_path.is_file():
+            return str(self.private_key_path), Path
+        if self.private_key:
+            try:
+                serialization.load_pem_private_key(
+                    data=str(self.private_key).encode("utf-8"), password=None
+                )
+            except Exception as e:
+                raise ValueError(f"failed to validate private key data: {e}") from e
+            return self.private_key, str
+
+        raise ValueError("private_key does not contain PEM private key or path")
+
+
+class SalesforceConnectionConfig(ConnectionConfig):
+    username: str
+    access_config: Secret[SalesforceAccessConfig]
+
+    @requires_dependencies(["simple_salesforce"], extras="salesforce")
+    def get_client(self) -> "Salesforce":
+        from simple_salesforce import Salesforce
+
+        access_config = self.access_config.get_secret_value()
+        pkey_value, pkey_type = access_config.get_private_key_value_and_type()
+
+        return Salesforce(
+            username=self.username,
+            consumer_key=access_config.consumer_key,
+            privatekey_file=pkey_value if pkey_type is Path else None,
+            privatekey=pkey_value if pkey_type is str else None,
+            version=SALESFORCE_API_VERSION,
+        )
+
+
+class SalesforceIndexerConfig(IndexerConfig):
+    categories: list[str]
+
+
+@dataclass
+class SalesforceIndexer(Indexer):
+    connection_config: SalesforceConnectionConfig
+    index_config: SalesforceIndexerConfig
+
+    def __post_init__(self):
+        for record_type in self.index_config.categories:
+            if record_type not in ACCEPTED_CATEGORIES:
+                raise ValueError(f"{record_type} not currently an accepted Salesforce category")
+
+    def precheck(self) -> None:
+        try:
+            self.connection_config.get_client()
+        except Exception as e:
+            logger.error(f"failed to validate connection: {e}", exc_info=True)
+            raise SourceConnectionError(f"failed to validate connection: {e}")
+
+    def get_file_extension(self, record_type) -> str:
+        if record_type == "EmailMessage":
+            extension = ".eml"
+        elif record_type in ["Account", "Lead", "Case", "Campaign"]:
+            extension = ".xml"
+        else:
+            raise MissingCategoryError(
+                f"There are no categories with the name: {record_type}",
+            )
+        return extension
+
+    @requires_dependencies(["simple_salesforce"], extras="salesforce")
+    def list_files(self) -> list[FileData]:
+        """Get Salesforce Ids for the records.
+        Send them to next phase where each doc gets downloaded into the
+        appropriate format for partitioning.
+        """
+        from simple_salesforce.exceptions import SalesforceMalformedRequest
+
+        client = self.connection_config.get_client()
+
+        files_list = []
+        for record_type in self.index_config.categories:
+            try:
+                # Get ids from Salesforce
+                records = client.query_all_iter(
+                    f"select Id, SystemModstamp, CreatedDate, LastModifiedDate from {record_type}",
+                )
+                for record in records:
+                    record_with_extension = record["Id"] + self.get_file_extension(
+                        record["attributes"]["type"]
+                    )
+                    source_identifiers = SourceIdentifiers(
+                        filename=record_with_extension,
+                        fullpath=f"{record['attributes']['type']}/{record_with_extension}",
+                    )
+                    files_list.append(
+                        FileData(
+                            connector_type=CONNECTOR_TYPE,
+                            identifier=record["Id"],
+                            source_identifiers=source_identifiers,
+                            metadata=FileDataSourceMetadata(
+                                url=record["attributes"]["url"],
+                                version=str(parser.parse(record["SystemModstamp"]).timestamp()),
+                                date_created=str(parser.parse(record["CreatedDate"]).timestamp()),
+                                date_modified=str(
+                                    parser.parse(record["LastModifiedDate"]).timestamp()
+                                ),
+                                record_locator={"id": record["Id"]},
+                            ),
+                            additional_metadata={"record_type": record["attributes"]["type"]},
+                            display_name=source_identifiers.fullpath,
+                        )
+                    )
+            except SalesforceMalformedRequest as e:
+                raise SalesforceMalformedRequest(f"Problem with Salesforce query: {e}")
+
+        return files_list
+
+    def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
+        for f in self.list_files():
+            yield f
+
+
+class SalesforceDownloaderConfig(DownloaderConfig):
+    pass
+
+
+@dataclass
+class SalesforceDownloader(Downloader):
+    connection_config: SalesforceConnectionConfig
+    download_config: SalesforceDownloaderConfig = field(
+        default_factory=lambda: SalesforceDownloaderConfig()
+    )
+    connector_type: str = CONNECTOR_TYPE
+
+    def _xml_for_record(self, record: OrderedDict) -> str:
+        """Creates partitionable xml file from a record"""
+        import xml.etree.ElementTree as ET
+
+        def create_xml_doc(data, parent, prefix=""):
+            for key, value in data.items():
+                if isinstance(value, OrderedDict):
+                    create_xml_doc(value, parent, prefix=f"{prefix}{key}.")
+                else:
+                    item = ET.Element("item")
+                    item.text = f"{prefix}{key}: {value}"
+                    parent.append(item)
+
+        root = ET.Element("root")
+        create_xml_doc(record, root)
+
+        xml_string = ET.tostring(root, encoding="utf-8", xml_declaration=True).decode()
+        return xml_string
+
+    def _eml_for_record(self, email_json: dict[str, Any]) -> str:
+        """Recreates standard expected .eml format using template."""
+        eml = EMAIL_TEMPLATE.substitute(
+            date=formatdate(parser.parse(email_json.get("MessageDate")).timestamp()),
+            message_identifier=email_json.get("MessageIdentifier"),
+            subject=email_json.get("Subject"),
+            from_email=email_json.get("FromAddress"),
+            to_email=email_json.get("ToAddress"),
+            textbody=email_json.get("TextBody"),
+            htmlbody=email_json.get("HtmlBody"),
+        )
+        return dedent(eml)
+
+    @SourceConnectionNetworkError.wrap
+    def _get_response(self, file_data: FileData) -> OrderedDict:
+        client = self.connection_config.get_client()
+        return client.query(
+            f"select FIELDS(STANDARD) from {file_data.additional_metadata['record_type']} where Id='{file_data.identifier}'",  # noqa: E501
+        )
+
+    def get_record(self, file_data: FileData) -> OrderedDict:
+        # Get record from Salesforce based on id
+        response = self._get_response(file_data)
+        logger.debug(f"response was returned for salesforce record id: {file_data.identifier}")
+        records = response["records"]
+        if not records:
+            raise ValueError(
+                f"No record found with record id {file_data.identifier}: {json.dumps(response)}"
+            )
+        record_json = records[0]
+        return record_json
+
+    def run(self, file_data: FileData, **kwargs: Any) -> DownloadResponse:
+        record = self.get_record(file_data)
+
+        try:
+            if file_data.additional_metadata["record_type"] == "EmailMessage":
+                document = self._eml_for_record(record)
+            else:
+                document = self._xml_for_record(record)
+            download_path = self.get_download_path(file_data=file_data)
+            download_path.parent.mkdir(parents=True, exist_ok=True)
+
+            with open(download_path, "w") as page_file:
+                page_file.write(document)
+
+        except Exception as e:
+            logger.error(f"failed to download file {file_data.identifier}: {e}", exc_info=True)
+            raise SourceConnectionNetworkError(f"failed to download file {file_data.identifier}")
+
+        return self.generate_download_response(file_data=file_data, download_path=download_path)
+
+
+salesforce_source_entry = SourceRegistryEntry(
+    connection_config=SalesforceConnectionConfig,
+    indexer_config=SalesforceIndexerConfig,
+    indexer=SalesforceIndexer,
+    downloader_config=SalesforceDownloaderConfig,
+    downloader=SalesforceDownloader,
+)

unstructured_ingest/processes/connectors/sharepoint.py

@@ -0,0 +1,282 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any, AsyncIterator, Optional
+
+from pydantic import Field
+
+from unstructured_ingest.data_types.file_data import (
+    FileData,
+)
+from unstructured_ingest.error import (
+    NotFoundError,
+    SourceConnectionError,
+    SourceConnectionNetworkError,
+    UserAuthError,
+    UserError,
+    ValueError,
+)
+from unstructured_ingest.logger import logger
+from unstructured_ingest.processes.connector_registry import (
+    SourceRegistryEntry,
+)
+from unstructured_ingest.processes.connectors.onedrive import (
+    OnedriveAccessConfig,
+    OnedriveConnectionConfig,
+    OnedriveDownloader,
+    OnedriveDownloaderConfig,
+    OnedriveIndexer,
+    OnedriveIndexerConfig,
+)
+from unstructured_ingest.utils.dep_check import requires_dependencies
+
+if TYPE_CHECKING:
+    from office365.onedrive.driveitems.driveItem import DriveItem
+    from office365.onedrive.sites.site import Site
+    from office365.runtime.client_request_exception import ClientRequestException
+
+CONNECTOR_TYPE = "sharepoint"
+LEGACY_DEFAULT_PATH = "Shared Documents"
+
+
+class SharepointAccessConfig(OnedriveAccessConfig):
+    client_cred: str = Field(description="Microsoft App client secret")
+
+
+class SharepointConnectionConfig(OnedriveConnectionConfig):
+    user_pname: Optional[str] = Field(
+        default=None,
+        description="User principal name or service account, usually your Azure AD email.",
+    )
+    site: str = Field(
+        description="Sharepoint site url. Process either base url e.g \
+                    https://[tenant].sharepoint.com  or relative sites \
+                    https://[tenant].sharepoint.com/sites/<site_name>. \
+                    To process all sites within the tenant pass a site url as \
+                    https://[tenant]-admin.sharepoint.com.\
+                    This requires the app to be registered at a tenant level"
+    )
+    library: Optional[str] = Field(
+        default=None,
+        description="Sharepoint library name. If not provided, the default \
+                    drive will be used.",
+    )
+
+    def _get_drive_item(self, client_site: Site) -> DriveItem:
+        """Helper method to get the drive item for the specified library or default drive."""
+        site_drive_item = None
+        if self.library:
+            for drive in client_site.drives.get().execute_query():
+                if drive.name == self.library:
+                    logger.info(f"Found the requested library: {self.library}")
+                    site_drive_item = drive.get().execute_query().root
+                    break
+
+        # If no specific library was found or requested, use the default drive
+        if not site_drive_item:
+            if self.library:
+                logger.warning(
+                    f"Library '{self.library}' not found in site '{self.site}'. "
+                    "Using the default drive instead."
+                )
+
+            site_drive_item = client_site.drive.get().execute_query().root
+
+        return site_drive_item
+
+
+class SharepointIndexerConfig(OnedriveIndexerConfig):
+    # TODO: We can probably make path non-optional on OnedriveIndexerConfig once tested
+    path: str = Field(default="")
+
+
+@dataclass
+class SharepointIndexer(OnedriveIndexer):
+    connection_config: SharepointConnectionConfig
+    index_config: SharepointIndexerConfig
+    connector_type: str = CONNECTOR_TYPE
+
+    def _handle_client_request_exception(self, e: ClientRequestException, context: str) -> None:
+        """Convert ClientRequestException to appropriate user-facing error based on HTTP status."""
+        if hasattr(e, "response") and e.response is not None and hasattr(e.response, "status_code"):
+            status_code = e.response.status_code
+            if status_code == 401:
+                raise UserAuthError(
+                    f"Unauthorized access to {context}. Check client credentials and permissions"
+                )
+            elif status_code == 403:
+                raise UserAuthError(
+                    f"Access forbidden to {context}. "
+                    f"Check app permissions (Sites.Read.All required)"
+                )
+            elif status_code == 404:
+                raise UserError(f"Not found: {context}")
+
+        raise UserError(f"Failed to access {context}: {str(e)}")
+
+    def _is_root_path(self, path: str) -> bool:
+        """Check if the path represents root access (empty string or legacy default)."""
+        return not path or not path.strip() or path == LEGACY_DEFAULT_PATH
+
+    def _get_target_drive_item(self, site_drive_item: DriveItem, path: str) -> DriveItem:
+        """Get the drive item to search in based on the path."""
+        if self._is_root_path(path):
+            return site_drive_item
+        else:
+            return site_drive_item.get_by_path(path).get().execute_query()
+
+    def _validate_folder_path(self, site_drive_item: DriveItem, path: str) -> None:
+        """Validate that a specific folder path exists and is accessible."""
+        from office365.runtime.client_request_exception import ClientRequestException
+
+        try:
+            path_item = site_drive_item.get_by_path(path).get().execute_query()
+            if path_item is None or not hasattr(path_item, "is_folder"):
+                raise UserError(
+                    f"SharePoint path '{path}' not found in site {self.connection_config.site}. "
+                    f"Check that the path exists and you have access to it"
+                )
+            logger.info(f"SharePoint folder path '{path}' validated successfully")
+        except ClientRequestException as e:
+            logger.error(f"Failed to access SharePoint path '{path}': {e}")
+            self._handle_client_request_exception(e, f"SharePoint path '{path}'")
+        except Exception as e:
+            logger.error(f"Unexpected error accessing SharePoint path '{path}': {e}")
+            raise UserError(f"Failed to validate SharePoint path '{path}': {str(e)}")
+
+    @requires_dependencies(["office365"], extras="sharepoint")
+    def precheck(self) -> None:
+        """Validate SharePoint connection before indexing."""
+        from office365.runtime.client_request_exception import ClientRequestException
+
+        # Validate authentication - this call will raise UserAuthError if invalid
+        self.connection_config.get_token()
+
+        try:
+            client = self.connection_config.get_client()
+            client_site = client.sites.get_by_url(self.connection_config.site).get().execute_query()
+            site_drive_item = self.connection_config._get_drive_item(client_site)
+
+            path = self.index_config.path
+            if not self._is_root_path(path):
+                self._validate_folder_path(site_drive_item, path)
+
+            logger.info(
+                f"SharePoint connection validated successfully for site: "
+                f"{self.connection_config.site}"
+            )
+
+        except ClientRequestException as e:
+            logger.error(f"SharePoint precheck failed for site: {self.connection_config.site}")
+            self._handle_client_request_exception(
+                e, f"SharePoint site {self.connection_config.site}"
+            )
+        except Exception as e:
+            logger.error(f"Unexpected error during SharePoint precheck: {e}", exc_info=True)
+            raise UserError(f"Failed to validate SharePoint connection: {str(e)}")
+
+    @requires_dependencies(["office365"], extras="sharepoint")
+    async def run_async(self, **kwargs: Any) -> AsyncIterator[FileData]:
+        from office365.runtime.client_request_exception import ClientRequestException
+
+        token_resp = await asyncio.to_thread(self.connection_config.get_token)
+        if "error" in token_resp:
+            raise SourceConnectionError(
+                f"[{self.connector_type}]: {token_resp['error']} "
+                f"({token_resp.get('error_description')})"
+            )
+
+        client = await asyncio.to_thread(self.connection_config.get_client)
+        try:
+            client_site = client.sites.get_by_url(self.connection_config.site).get().execute_query()
+            site_drive_item = self.connection_config._get_drive_item(client_site)
+        except ClientRequestException as e:
+            logger.error(f"Failed to access SharePoint site: {self.connection_config.site}")
+            raise SourceConnectionError(
+                f"Unable to access SharePoint site at {self.connection_config.site}: {str(e)}"
+            )
+
+        path = self.index_config.path
+        target_drive_item = await asyncio.to_thread(
+            self._get_target_drive_item, site_drive_item, path
+        )
+
+        for drive_item in target_drive_item.get_files(
+            recursive=self.index_config.recursive
+        ).execute_query():
+            file_data = await self.drive_item_to_file_data(drive_item=drive_item)
+            yield file_data
+
+
+class SharepointDownloaderConfig(OnedriveDownloaderConfig):
+    max_retries: int = 10
+
+
+@dataclass
+class SharepointDownloader(OnedriveDownloader):
+    connection_config: SharepointConnectionConfig
+    download_config: SharepointDownloaderConfig
+    connector_type: str = CONNECTOR_TYPE
+
+    @staticmethod
+    def retry_on_status_code(exc):
+        error_msg = str(exc).lower()
+        return "429" in error_msg or "activitylimitreached" in error_msg or "throttled" in error_msg
+
+    @SourceConnectionNetworkError.wrap
+    @requires_dependencies(["office365"], extras="sharepoint")
+    def _fetch_file(self, file_data: FileData) -> DriveItem:
+        from office365.runtime.client_request_exception import ClientRequestException
+        from tenacity import (
+            before_log,
+            retry,
+            retry_if_exception,
+            stop_after_attempt,
+            wait_exponential,
+        )
+
+        if file_data.source_identifiers is None or not file_data.source_identifiers.fullpath:
+            raise ValueError(
+                f"file data doesn't have enough information to get "
+                f"file content: {file_data.model_dump()}"
+            )
+
+        server_relative_path = file_data.source_identifiers.fullpath
+        client = self.connection_config.get_client()
+
+        @retry(
+            stop=stop_after_attempt(self.download_config.max_retries),
+            wait=wait_exponential(exp_base=2, multiplier=1, min=2, max=10),
+            retry=retry_if_exception(self.retry_on_status_code),
+            before=before_log(logger, logging.DEBUG),
+            reraise=True,
+        )
+        def _get_item_by_path() -> DriveItem:
+            try:
+                client_site = (
+                    client.sites.get_by_url(self.connection_config.site).get().execute_query()
+                )
+                site_drive_item = self.connection_config._get_drive_item(client_site)
+            except ClientRequestException:
+                logger.info(f"Site not found: {self.connection_config.site}")
+                raise SourceConnectionError(f"Site not found: {self.connection_config.site}")
+            file = site_drive_item.get_by_path(server_relative_path).get().execute_query()
+            return file
+
+        # Call the retry-wrapped function
+        file = _get_item_by_path()
+
+        if not file:
+            raise NotFoundError(f"file not found: {server_relative_path}")
+        return file
+
+
+sharepoint_source_entry = SourceRegistryEntry(
+    connection_config=SharepointConnectionConfig,
+    indexer_config=SharepointIndexerConfig,
+    indexer=SharepointIndexer,
+    downloader_config=SharepointDownloaderConfig,
+    downloader=SharepointDownloader,
+)