unstructured-ingest 0.0.24__py3-none-any.whl → 0.1.0__py3-none-any.whl

unstructured_ingest/v2/processes/connectors/outlook.py

@@ -0,0 +1,239 @@
+import hashlib
+import time
+from dataclasses import dataclass, field
+from datetime import timezone
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Coroutine, Generator
+
+from pydantic import Field, Secret
+
+from unstructured_ingest.error import SourceConnectionError
+from unstructured_ingest.logger import logger
+from unstructured_ingest.utils.dep_check import requires_dependencies
+from unstructured_ingest.v2.interfaces import (
+    AccessConfig,
+    ConnectionConfig,
+    Downloader,
+    DownloaderConfig,
+    FileData,
+    Indexer,
+    IndexerConfig,
+    download_responses,
+)
+from unstructured_ingest.v2.interfaces.file_data import FileDataSourceMetadata, SourceIdentifiers
+from unstructured_ingest.v2.processes.connector_registry import SourceRegistryEntry
+
+MAX_EMAILS_PER_FOLDER = 1_000_000  # Maximum number of emails per folder
+
+if TYPE_CHECKING:
+    from office365.graph_client import GraphClient
+    from office365.outlook.mail.folders.folder import MailFolder
+    from office365.outlook.mail.messages.message import Message
+
+
+CONNECTOR_TYPE = "outlook"
+
+
+class OutlookAccessConfig(AccessConfig):
+    client_credential: str = Field(description="Azure AD App client secret", alias="client_cred")
+
+
+class OutlookConnectionConfig(ConnectionConfig):
+    access_config: Secret[OutlookAccessConfig]
+    client_id: str = Field(description="Azure AD App client ID")
+    tenant: str = Field(
+        default="common", description="ID or domain name associated with your Azure AD instance"
+    )
+    authority_url: str = Field(
+        default="https://login.microsoftonline.com",
+        description="Authentication token provider for Microsoft apps",
+    )
+
+    @requires_dependencies(["msal"], extras="outlook")
+    def _acquire_token(self):
+        """Acquire token via MSAL"""
+        from msal import ConfidentialClientApplication
+
+        # NOTE: It'd be nice to use `msal.authority.AuthorityBuilder` here paired with AZURE_PUBLIC
+        # constant as default in the future but they do not fit well with `authority_url` right now
+        authority_url = f"{self.authority_url.rstrip('/')}/{self.tenant}"
+        app = ConfidentialClientApplication(
+            authority=authority_url,
+            client_id=self.client_id,
+            client_credential=self.access_config.get_secret_value().client_credential,
+        )
+        token = app.acquire_token_for_client(scopes=["https://graph.microsoft.com/.default"])
+        return token
+
+    @requires_dependencies(["office365"], extras="outlook")
+    @SourceConnectionError.wrap
+    def get_client(self) -> "GraphClient":
+        from office365.graph_client import GraphClient
+
+        return GraphClient(self._acquire_token)
+
+
+class OutlookIndexerConfig(IndexerConfig):
+    outlook_folders: list[str] = Field(
+        description="Folders to download email messages from. Do not specify subfolders. "
+        "Use quotes if there are spaces in folder names."
+    )
+    recursive: bool = Field(
+        default=False,
+        description="Recursively download files in their respective folders otherwise stop at the"
+        " files in provided folder level.",
+    )
+    user_email: str = Field(description="Outlook email to download messages from.")
+
+
+@dataclass
+class OutlookIndexer(Indexer):
+    index_config: OutlookIndexerConfig
+    connection_config: OutlookConnectionConfig
+    connector_type: str = CONNECTOR_TYPE
+
+    def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
+        messages = self._list_messages(recursive=self.index_config.recursive)
+
+        for message in messages:
+            yield self._message_to_file_data(message)
+
+    def run_async(self, **kwargs: Any) -> Coroutine[Any, Any, Any]:
+        raise NotImplementedError
+
+    @SourceConnectionError.wrap
+    def precheck(self) -> None:
+        client = self.connection_config.get_client()
+        client.users[self.index_config.user_email].get().execute_query()
+
+    def is_async(self) -> bool:
+        return False
+
+    def _list_messages(self, recursive: bool) -> list["Message"]:
+        mail_folders = self._get_selected_root_folders()
+        messages = []
+
+        while mail_folders:
+            mail_folder = mail_folders.pop()
+            messages += list(mail_folder.messages.get().top(MAX_EMAILS_PER_FOLDER).execute_query())
+
+            if recursive:
+                mail_folders += list(mail_folder.child_folders.get().execute_query())
+
+        return messages
+
+    def _get_selected_root_folders(self) -> list["MailFolder"]:
+        client_user = self.connection_config.get_client().users[self.index_config.user_email]
+        root_mail_folders = client_user.mail_folders.get().execute_query()
+
+        selected_names_normalized = [
+            folder_name.lower() for folder_name in self.index_config.outlook_folders
+        ]
+        selected_root_mail_folders = [
+            folder
+            for folder in root_mail_folders
+            if folder.display_name.lower() in selected_names_normalized
+        ]
+
+        if not selected_root_mail_folders:
+            logger.error(
+                f"Root folders selected in configuration: {self.index_config.outlook_folders}"
+                f"not found for user email {self.index_config.user_email}. Aborting."
+            )
+            raise ValueError("Root folders selected in configuration not found.")
+
+        return selected_root_mail_folders
+
+    def _message_to_file_data(self, message: "Message") -> FileData:
+        fullpath = self._generate_fullpath(message)
+
+        return FileData(
+            identifier=message.id,
+            connector_type=CONNECTOR_TYPE,
+            source_identifiers=SourceIdentifiers(filename=fullpath.name, fullpath=str(fullpath)),
+            metadata=FileDataSourceMetadata(
+                url=message.resource_url,
+                version=message.change_key,
+                date_modified=str(
+                    message.last_modified_datetime.replace(tzinfo=timezone.utc).timestamp()
+                ),
+                date_created=str(message.created_datetime.replace(tzinfo=timezone.utc).timestamp()),
+                date_processed=str(time.time()),
+                record_locator={
+                    "message_id": message.id,
+                    "user_email": self.index_config.user_email,
+                },
+            ),
+            additional_metadata={
+                "sent_from": str(message.sent_from),
+                "to_recipients": [str(recipient) for recipient in message.to_recipients],
+                "bcc_recipients": [str(recipient) for recipient in message.to_recipients],
+                "subject": message.subject,
+                "conversation_id": message.conversation_id,
+                "is_draft": message.is_draft,
+                "is_read": message.is_read,
+                "has_attachments": message.has_attachments,
+                "importance": message.importance,
+            },
+        )
+
+    def _generate_fullpath(self, message: "Message") -> Path:
+        return Path(hashlib.sha256(message.id.encode("utf-8")).hexdigest()[:16] + ".eml")
+
+
+class OutlookDownloaderConfig(DownloaderConfig):
+    pass
+
+
+@dataclass
+class OutlookDownloader(Downloader):
+    connector_type: str = CONNECTOR_TYPE
+    connection_config: OutlookConnectionConfig
+    download_config: OutlookDownloaderConfig = field(default_factory=OutlookDownloaderConfig)
+
+    def run(self, file_data: FileData, **kwargs: Any) -> download_responses:
+        # NOTE: Indexer should provide source identifiers required to generate the download path
+        download_path = self.get_download_path(file_data)
+        if download_path is None:
+            logger.error(
+                "Generated download path is None, source_identifiers might be missing"
+                "from FileData."
+            )
+            raise ValueError("Generated invalid download path.")
+
+        self._download_message(file_data, download_path)
+        return self.generate_download_response(file_data, download_path)
+
+    def is_async(self) -> bool:
+        return False
+
+    def _download_message(self, file_data: FileData, download_path: Path) -> None:
+        # NOTE: Indexer should supply the record locator in metadata
+        if (
+            file_data.metadata.record_locator is None
+            or "user_email" not in file_data.metadata.record_locator
+            or "message_id" not in file_data.metadata.record_locator
+        ):
+            logger.error(
+                f"Invalid record locator in metadata: {file_data.metadata.record_locator}."
+                "Keys 'user_email' and 'message_id' must be present."
+            )
+            raise ValueError("Invalid record locator.")
+
+        user_email = file_data.metadata.record_locator["user_email"]
+        message_id = file_data.metadata.record_locator["message_id"]
+
+        message = self.connection_config.get_client().users[user_email].messages[message_id]
+        download_path.parent.mkdir(exist_ok=True, parents=True)
+
+        with open(download_path, "wb") as file:
+            message.download(file).execute_query()
+
+
+outlook_source_entry = SourceRegistryEntry(
+    indexer=OutlookIndexer,
+    indexer_config=OutlookIndexerConfig,
+    downloader=OutlookDownloader,
+    downloader_config=OutlookDownloaderConfig,
+    connection_config=OutlookConnectionConfig,
+)

unstructured_ingest/v2/processes/connectors/pinecone.py

@@ -94,6 +94,10 @@ class PineconeUploaderConfig(UploaderConfig):
     pool_threads: Optional[int] = Field(
         default=1, description="Optional limit on number of threads to use for upload"
     )
+    namespace: Optional[str] = Field(
+        default=None,
+        description="The namespace to write to. If not specified, the default namespace is used",
+    )
 
 
 @dataclass
@@ -183,7 +187,11 @@ class PineconeUploader(Uploader):
             pool_threads = max_pool_threads
         index = self.connection_config.get_index(pool_threads=pool_threads)
         with index:
-            async_results = [index.upsert(vectors=chunk, async_req=True) for chunk in chunks]
+            upsert_kwargs = [{"vectors": chunk, "async_req": True} for chunk in chunks]
+            if namespace := self.upload_config.namespace:
+                for kwargs in upsert_kwargs:
+                    kwargs["namespace"] = namespace
+            async_results = [index.upsert(**kwarg) for kwarg in upsert_kwargs]
             # Wait for and retrieve responses (this raises in case of error)
             try:
                 results = [async_result.get() for async_result in async_results]

unstructured_ingest/v2/processes/connectors/sql/__init__.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+from unstructured_ingest.v2.processes.connector_registry import (
+    add_destination_entry,
+)
+
+from .postgres import CONNECTOR_TYPE as POSTGRES_CONNECTOR_TYPE
+from .postgres import postgres_destination_entry
+from .sqlite import CONNECTOR_TYPE as SQLITE_CONNECTOR_TYPE
+from .sqlite import sqlite_destination_entry
+
+add_destination_entry(destination_type=SQLITE_CONNECTOR_TYPE, entry=sqlite_destination_entry)
+add_destination_entry(destination_type=POSTGRES_CONNECTOR_TYPE, entry=postgres_destination_entry)

unstructured_ingest/v2/processes/connectors/sql/postgres.py

@@ -0,0 +1,121 @@
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Optional
+
+import numpy as np
+import pandas as pd
+from pydantic import Field, Secret
+
+from unstructured_ingest.utils.dep_check import requires_dependencies
+from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.v2.processes.connector_registry import DestinationRegistryEntry
+from unstructured_ingest.v2.processes.connectors.sql.sql import (
+    _DATE_COLUMNS,
+    SQLAccessConfig,
+    SQLConnectionConfig,
+    SQLUploader,
+    SQLUploaderConfig,
+    SQLUploadStager,
+    SQLUploadStagerConfig,
+    parse_date_string,
+)
+
+if TYPE_CHECKING:
+    from psycopg2.extensions import connection as PostgresConnection
+
+CONNECTOR_TYPE = "postgres"
+
+
+class PostgresAccessConfig(SQLAccessConfig):
+    password: Optional[str] = Field(default=None, description="DB password")
+
+
+class PostgresConnectionConfig(SQLConnectionConfig):
+    access_config: Secret[PostgresAccessConfig] = Field(
+        default=PostgresAccessConfig(), validate_default=True
+    )
+    database: Optional[str] = Field(
+        default=None,
+        description="Database name.",
+    )
+    username: Optional[str] = Field(default=None, description="DB username")
+    host: Optional[str] = Field(default=None, description="DB host")
+    port: Optional[int] = Field(default=5432, description="DB host connection port")
+    connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
+
+    @requires_dependencies(["psycopg2"], extras="postgres")
+    def get_connection(self) -> "PostgresConnection":
+        from psycopg2 import connect
+
+        access_config = self.access_config.get_secret_value()
+        return connect(
+            user=self.username,
+            password=access_config.password,
+            dbname=self.database,
+            host=self.host,
+            port=self.port,
+        )
+
+
+class PostgresUploadStagerConfig(SQLUploadStagerConfig):
+    pass
+
+
+class PostgresUploadStager(SQLUploadStager):
+    upload_stager_config: PostgresUploadStagerConfig
+
+
+class PostgresUploaderConfig(SQLUploaderConfig):
+    pass
+
+
+@dataclass
+class PostgresUploader(SQLUploader):
+    upload_config: PostgresUploaderConfig = field(default_factory=PostgresUploaderConfig)
+    connection_config: PostgresConnectionConfig
+    connector_type: str = CONNECTOR_TYPE
+
+    def prepare_data(
+        self, columns: list[str], data: tuple[tuple[Any, ...], ...]
+    ) -> list[tuple[Any, ...]]:
+        output = []
+        for row in data:
+            parsed = []
+            for column_name, value in zip(columns, row):
+                if column_name in _DATE_COLUMNS:
+                    if value is None:
+                        parsed.append(None)
+                    else:
+                        parsed.append(parse_date_string(value))
+                else:
+                    parsed.append(value)
+            output.append(tuple(parsed))
+        return output
+
+    def upload_contents(self, path: Path) -> None:
+        df = pd.read_json(path, orient="records", lines=True)
+        logger.debug(f"uploading {len(df)} entries to {self.connection_config.database} ")
+        df.replace({np.nan: None}, inplace=True)
+
+        columns = tuple(df.columns)
+        stmt = f"INSERT INTO {self.upload_config.table_name} ({','.join(columns)}) \
+                VALUES({','.join(['%s' for x in columns])})"  # noqa E501
+
+        for rows in pd.read_json(
+            path, orient="records", lines=True, chunksize=self.upload_config.batch_size
+        ):
+            with self.connection_config.get_connection() as conn:
+                values = self.prepare_data(columns, tuple(rows.itertuples(index=False, name=None)))
+                with conn.cursor() as cur:
+                    cur.executemany(stmt, values)
+
+                conn.commit()
+
+
+postgres_destination_entry = DestinationRegistryEntry(
+    connection_config=PostgresConnectionConfig,
+    uploader=PostgresUploader,
+    uploader_config=PostgresUploaderConfig,
+    upload_stager=PostgresUploadStager,
+    upload_stager_config=PostgresUploadStagerConfig,
+)

unstructured_ingest/v2/processes/connectors/sql/sql.py

@@ -0,0 +1,181 @@
+import json
+import uuid
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from datetime import date, datetime
+from pathlib import Path
+from typing import Any, Union
+
+import pandas as pd
+from dateutil import parser
+from pydantic import Field, Secret
+
+from unstructured_ingest.error import DestinationConnectionError
+from unstructured_ingest.v2.interfaces import (
+    AccessConfig,
+    ConnectionConfig,
+    FileData,
+    Uploader,
+    UploaderConfig,
+    UploadStager,
+    UploadStagerConfig,
+)
+from unstructured_ingest.v2.logger import logger
+
+_COLUMNS = (
+    "id",
+    "element_id",
+    "text",
+    "embeddings",
+    "type",
+    "system",
+    "layout_width",
+    "layout_height",
+    "points",
+    "url",
+    "version",
+    "date_created",
+    "date_modified",
+    "date_processed",
+    "permissions_data",
+    "record_locator",
+    "category_depth",
+    "parent_id",
+    "attached_filename",
+    "filetype",
+    "last_modified",
+    "file_directory",
+    "filename",
+    "languages",
+    "page_number",
+    "links",
+    "page_name",
+    "link_urls",
+    "link_texts",
+    "sent_from",
+    "sent_to",
+    "subject",
+    "section",
+    "header_footer_type",
+    "emphasized_text_contents",
+    "emphasized_text_tags",
+    "text_as_html",
+    "regex_metadata",
+    "detection_class_prob",
+)
+
+_DATE_COLUMNS = ("date_created", "date_modified", "date_processed", "last_modified")
+
+
+def parse_date_string(date_value: Union[str, int]) -> date:
+    try:
+        timestamp = float(date_value) / 1000 if isinstance(date_value, int) else float(date_value)
+        return datetime.fromtimestamp(timestamp)
+    except Exception as e:
+        logger.debug(f"date {date_value} string not a timestamp: {e}")
+    return parser.parse(date_value)
+
+
+class SQLAccessConfig(AccessConfig):
+    pass
+
+
+class SQLConnectionConfig(ConnectionConfig, ABC):
+    access_config: Secret[SQLAccessConfig] = Field(default=SQLAccessConfig(), validate_default=True)
+
+    @abstractmethod
+    def get_connection(self) -> Any:
+        pass
+
+
+class SQLUploadStagerConfig(UploadStagerConfig):
+    pass
+
+
+@dataclass
+class SQLUploadStager(UploadStager):
+    upload_stager_config: SQLUploadStagerConfig = field(default_factory=SQLUploadStagerConfig)
+
+    def run(
+        self,
+        elements_filepath: Path,
+        file_data: FileData,
+        output_dir: Path,
+        output_filename: str,
+        **kwargs: Any,
+    ) -> Path:
+        with open(elements_filepath) as elements_file:
+            elements_contents: list[dict] = json.load(elements_file)
+        output_path = Path(output_dir) / Path(f"{output_filename}.json")
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+
+        output = []
+        for data in elements_contents:
+            metadata: dict[str, Any] = data.pop("metadata", {})
+            data_source = metadata.pop("data_source", {})
+            coordinates = metadata.pop("coordinates", {})
+
+            data.update(metadata)
+            data.update(data_source)
+            data.update(coordinates)
+
+            data["id"] = str(uuid.uuid4())
+
+            # remove extraneous, not supported columns
+            data = {k: v for k, v in data.items() if k in _COLUMNS}
+
+            output.append(data)
+
+        df = pd.DataFrame.from_dict(output)
+        for column in filter(lambda x: x in df.columns, _DATE_COLUMNS):
+            df[column] = df[column].apply(parse_date_string)
+        for column in filter(
+            lambda x: x in df.columns,
+            ("permissions_data", "record_locator", "points", "links"),
+        ):
+            df[column] = df[column].apply(
+                lambda x: json.dumps(x) if isinstance(x, (list, dict)) else None
+            )
+        for column in filter(
+            lambda x: x in df.columns,
+            ("version", "page_number", "regex_metadata"),
+        ):
+            df[column] = df[column].apply(str)
+
+        with output_path.open("w") as output_file:
+            df.to_json(output_file, orient="records", lines=True)
+        return output_path
+
+
+class SQLUploaderConfig(UploaderConfig):
+    batch_size: int = Field(default=50, description="Number of records per batch")
+    table_name: str = Field(default="elements", description="which table to upload contents to")
+
+
+@dataclass
+class SQLUploader(Uploader):
+    upload_config: SQLUploaderConfig
+    connection_config: SQLConnectionConfig
+
+    def precheck(self) -> None:
+        try:
+            connection = self.connection_config.get_connection()
+            cursor = connection.cursor()
+            cursor.execute("SELECT 1;")
+            cursor.close()
+        except Exception as e:
+            logger.error(f"failed to validate connection: {e}", exc_info=True)
+            raise DestinationConnectionError(f"failed to validate connection: {e}")
+
+    @abstractmethod
+    def prepare_data(
+        self, columns: list[str], data: tuple[tuple[Any, ...], ...]
+    ) -> list[tuple[Any, ...]]:
+        pass
+
+    @abstractmethod
+    def upload_contents(self, path: Path) -> None:
+        pass
+
+    def run(self, path: Path, file_data: FileData, **kwargs: Any) -> None:
+        self.upload_contents(path=path)

unstructured_ingest/v2/processes/connectors/sql/sqlite.py

@@ -0,0 +1,109 @@
+import json
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+import numpy as np
+import pandas as pd
+from pydantic import Field, Secret
+
+from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.v2.processes.connector_registry import DestinationRegistryEntry
+from unstructured_ingest.v2.processes.connectors.sql.sql import (
+    _DATE_COLUMNS,
+    SQLAccessConfig,
+    SQLConnectionConfig,
+    SQLUploader,
+    SQLUploaderConfig,
+    SQLUploadStager,
+    SQLUploadStagerConfig,
+    parse_date_string,
+)
+
+if TYPE_CHECKING:
+    from sqlite3 import Connection as SqliteConnection
+
+CONNECTOR_TYPE = "sqlite"
+
+
+class SQLiteAccessConfig(SQLAccessConfig):
+    pass
+
+
+class SQLiteConnectionConfig(SQLConnectionConfig):
+    access_config: Secret[SQLiteAccessConfig] = Field(
+        default=SQLiteAccessConfig(), validate_default=True
+    )
+    database_path: Path = Field(
+        description="Path to the .db file.",
+    )
+    connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
+
+    def get_connection(self) -> "SqliteConnection":
+        from sqlite3 import connect
+
+        return connect(database=self.database_path)
+
+
+class SQLiteUploadStagerConfig(SQLUploadStagerConfig):
+    pass
+
+
+class SQLiteUploadStager(SQLUploadStager):
+    upload_stager_config: SQLiteUploadStagerConfig
+
+
+class SQLiteUploaderConfig(SQLUploaderConfig):
+    pass
+
+
+@dataclass
+class SQLiteUploader(SQLUploader):
+    upload_config: SQLiteUploaderConfig = field(default_factory=SQLiteUploaderConfig)
+    connection_config: SQLiteConnectionConfig
+    connector_type: str = CONNECTOR_TYPE
+
+    def prepare_data(
+        self, columns: list[str], data: tuple[tuple[Any, ...], ...]
+    ) -> list[tuple[Any, ...]]:
+        output = []
+        for row in data:
+            parsed = []
+            for column_name, value in zip(columns, row):
+                if isinstance(value, (list, dict)):
+                    value = json.dumps(value)
+                if column_name in _DATE_COLUMNS:
+                    if value is None:
+                        parsed.append(None)
+                    else:
+                        parsed.append(parse_date_string(value))
+                else:
+                    parsed.append(value)
+            output.append(tuple(parsed))
+        return output
+
+    def upload_contents(self, path: Path) -> None:
+        df = pd.read_json(path, orient="records", lines=True)
+        logger.debug(f"uploading {len(df)} entries to {self.connection_config.database_path} ")
+        df.replace({np.nan: None}, inplace=True)
+
+        columns = tuple(df.columns)
+        stmt = f"INSERT INTO {self.upload_config.table_name} ({','.join(columns)}) \
+                VALUES({','.join(['?' for x in columns])})"  # noqa E501
+
+        for rows in pd.read_json(
+            path, orient="records", lines=True, chunksize=self.upload_config.batch_size
+        ):
+            with self.connection_config.get_connection() as conn:
+                values = self.prepare_data(columns, tuple(rows.itertuples(index=False, name=None)))
+                conn.executemany(stmt, values)
+                conn.commit()
+
+
+sqlite_destination_entry = DestinationRegistryEntry(
+    connection_config=SQLiteConnectionConfig,
+    uploader=SQLiteUploader,
+    uploader_config=SQLiteUploaderConfig,
+    upload_stager=SQLiteUploadStager,
+    upload_stager_config=SQLiteUploadStagerConfig,
+)