unstructured-ingest 0.0.25__py3-none-any.whl → 0.1.1__py3-none-any.whl

unstructured_ingest/v2/processes/connectors/sql/sql.py

@@ -0,0 +1,310 @@
+import hashlib
+import json
+import sys
+import uuid
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field, replace
+from datetime import date, datetime
+from pathlib import Path
+from time import time
+from typing import Any, Generator, Union
+
+import pandas as pd
+from dateutil import parser
+from pydantic import Field, Secret
+
+from unstructured_ingest.error import DestinationConnectionError, SourceConnectionError
+from unstructured_ingest.v2.interfaces import (
+    AccessConfig,
+    ConnectionConfig,
+    Downloader,
+    DownloaderConfig,
+    DownloadResponse,
+    FileData,
+    FileDataSourceMetadata,
+    Indexer,
+    IndexerConfig,
+    Uploader,
+    UploaderConfig,
+    UploadStager,
+    UploadStagerConfig,
+    download_responses,
+)
+from unstructured_ingest.v2.logger import logger
+
+_COLUMNS = (
+    "id",
+    "element_id",
+    "text",
+    "embeddings",
+    "type",
+    "system",
+    "layout_width",
+    "layout_height",
+    "points",
+    "url",
+    "version",
+    "date_created",
+    "date_modified",
+    "date_processed",
+    "permissions_data",
+    "record_locator",
+    "category_depth",
+    "parent_id",
+    "attached_filename",
+    "filetype",
+    "last_modified",
+    "file_directory",
+    "filename",
+    "languages",
+    "page_number",
+    "links",
+    "page_name",
+    "link_urls",
+    "link_texts",
+    "sent_from",
+    "sent_to",
+    "subject",
+    "section",
+    "header_footer_type",
+    "emphasized_text_contents",
+    "emphasized_text_tags",
+    "text_as_html",
+    "regex_metadata",
+    "detection_class_prob",
+)
+
+_DATE_COLUMNS = ("date_created", "date_modified", "date_processed", "last_modified")
+
+
+def parse_date_string(date_value: Union[str, int]) -> date:
+    try:
+        timestamp = float(date_value) / 1000 if isinstance(date_value, int) else float(date_value)
+        return datetime.fromtimestamp(timestamp)
+    except Exception as e:
+        logger.debug(f"date {date_value} string not a timestamp: {e}")
+    return parser.parse(date_value)
+
+
+class SQLAccessConfig(AccessConfig):
+    pass
+
+
+class SQLConnectionConfig(ConnectionConfig, ABC):
+    access_config: Secret[SQLAccessConfig] = Field(default=SQLAccessConfig(), validate_default=True)
+
+    @abstractmethod
+    def get_connection(self) -> Any:
+        pass
+
+
+class SQLIndexerConfig(IndexerConfig):
+    table_name: str
+    id_column: str
+    batch_size: int = 100
+
+
+class SQLIndexer(Indexer, ABC):
+    connection_config: SQLConnectionConfig
+    index_config: SQLIndexerConfig
+
+    @abstractmethod
+    def _get_doc_ids(self) -> list[str]:
+        pass
+
+    def precheck(self) -> None:
+        try:
+            connection = self.connection_config.get_connection()
+            cursor = connection.cursor()
+            cursor.execute("SELECT 1;")
+            cursor.close()
+        except Exception as e:
+            logger.error(f"failed to validate connection: {e}", exc_info=True)
+            raise SourceConnectionError(f"failed to validate connection: {e}")
+
+    def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
+        ids = self._get_doc_ids()
+        id_batches: list[frozenset[str]] = [
+            frozenset(
+                ids[
+                    i
+                    * self.index_config.batch_size : (i + 1)  # noqa
+                    * self.index_config.batch_size
+                ]
+            )
+            for i in range(
+                (len(ids) + self.index_config.batch_size - 1) // self.index_config.batch_size
+            )
+        ]
+        for batch in id_batches:
+            # Make sure the hash is always a positive number to create identified
+            identified = str(hash(batch) + sys.maxsize + 1)
+            yield FileData(
+                identifier=identified,
+                connector_type=self.connector_type,
+                metadata=FileDataSourceMetadata(
+                    date_processed=str(time()),
+                ),
+                doc_type="batch",
+                additional_metadata={
+                    "ids": list(batch),
+                    "table_name": self.index_config.table_name,
+                    "id_column": self.index_config.id_column,
+                },
+            )
+
+
+class SQLDownloaderConfig(DownloaderConfig):
+    fields: list[str] = field(default_factory=list)
+
+
+class SQLDownloader(Downloader, ABC):
+    connection_config: SQLConnectionConfig
+    download_config: SQLDownloaderConfig
+
+    @abstractmethod
+    def query_db(self, file_data: FileData) -> tuple[list[tuple], list[str]]:
+        pass
+
+    def sql_to_df(self, rows: list[tuple], columns: list[str]) -> list[pd.DataFrame]:
+        data = [dict(zip(columns, row)) for row in rows]
+        df = pd.DataFrame(data)
+        dfs = [pd.DataFrame([row.values], columns=df.columns) for index, row in df.iterrows()]
+        return dfs
+
+    def get_data(self, file_data: FileData) -> list[pd.DataFrame]:
+        rows, columns = self.query_db(file_data=file_data)
+        return self.sql_to_df(rows=rows, columns=columns)
+
+    def get_identifier(self, table_name: str, record_id: str) -> str:
+        f = f"{table_name}-{record_id}"
+        if self.download_config.fields:
+            f = "{}-{}".format(
+                f,
+                hashlib.sha256(",".join(self.download_config.fields).encode()).hexdigest()[:8],
+            )
+        return f
+
+    def generate_download_response(
+        self, result: pd.DataFrame, file_data: FileData
+    ) -> DownloadResponse:
+        id_column = file_data.additional_metadata["id_column"]
+        table_name = file_data.additional_metadata["table_name"]
+        record_id = result.iloc[0][id_column]
+        filename_id = self.get_identifier(table_name=table_name, record_id=record_id)
+        filename = f"{filename_id}.csv"
+        download_path = self.download_dir / Path(filename)
+        logger.debug(
+            f"Downloading results from table {table_name} and id {record_id} to {download_path}"
+        )
+        download_path.parent.mkdir(parents=True, exist_ok=True)
+        result.to_csv(download_path)
+        copied_file_data = replace(file_data)
+        copied_file_data.identifier = filename_id
+        copied_file_data.doc_type = "file"
+        copied_file_data.additional_metadata.pop("ids", None)
+        return super().generate_download_response(
+            file_data=copied_file_data, download_path=download_path
+        )
+
+    def run(self, file_data: FileData, **kwargs: Any) -> download_responses:
+        data_dfs = self.get_data(file_data=file_data)
+        download_responses = []
+        for df in data_dfs:
+            download_responses.append(
+                self.generate_download_response(result=df, file_data=file_data)
+            )
+        return download_responses
+
+
+class SQLUploadStagerConfig(UploadStagerConfig):
+    pass
+
+
+@dataclass
+class SQLUploadStager(UploadStager):
+    upload_stager_config: SQLUploadStagerConfig = field(default_factory=SQLUploadStagerConfig)
+
+    def run(
+        self,
+        elements_filepath: Path,
+        file_data: FileData,
+        output_dir: Path,
+        output_filename: str,
+        **kwargs: Any,
+    ) -> Path:
+        with open(elements_filepath) as elements_file:
+            elements_contents: list[dict] = json.load(elements_file)
+        output_path = Path(output_dir) / Path(f"{output_filename}.json")
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+
+        output = []
+        for data in elements_contents:
+            metadata: dict[str, Any] = data.pop("metadata", {})
+            data_source = metadata.pop("data_source", {})
+            coordinates = metadata.pop("coordinates", {})
+
+            data.update(metadata)
+            data.update(data_source)
+            data.update(coordinates)
+
+            data["id"] = str(uuid.uuid4())
+
+            # remove extraneous, not supported columns
+            data = {k: v for k, v in data.items() if k in _COLUMNS}
+
+            output.append(data)
+
+        df = pd.DataFrame.from_dict(output)
+        for column in filter(lambda x: x in df.columns, _DATE_COLUMNS):
+            df[column] = df[column].apply(parse_date_string)
+        for column in filter(
+            lambda x: x in df.columns,
+            ("permissions_data", "record_locator", "points", "links"),
+        ):
+            df[column] = df[column].apply(
+                lambda x: json.dumps(x) if isinstance(x, (list, dict)) else None
+            )
+        for column in filter(
+            lambda x: x in df.columns,
+            ("version", "page_number", "regex_metadata"),
+        ):
+            df[column] = df[column].apply(str)
+
+        with output_path.open("w") as output_file:
+            df.to_json(output_file, orient="records", lines=True)
+        return output_path
+
+
+class SQLUploaderConfig(UploaderConfig):
+    batch_size: int = Field(default=50, description="Number of records per batch")
+    table_name: str = Field(default="elements", description="which table to upload contents to")
+
+
+@dataclass
+class SQLUploader(Uploader):
+    upload_config: SQLUploaderConfig
+    connection_config: SQLConnectionConfig
+
+    def precheck(self) -> None:
+        try:
+            connection = self.connection_config.get_connection()
+            cursor = connection.cursor()
+            cursor.execute("SELECT 1;")
+            cursor.close()
+        except Exception as e:
+            logger.error(f"failed to validate connection: {e}", exc_info=True)
+            raise DestinationConnectionError(f"failed to validate connection: {e}")
+
+    @abstractmethod
+    def prepare_data(
+        self, columns: list[str], data: tuple[tuple[Any, ...], ...]
+    ) -> list[tuple[Any, ...]]:
+        pass
+
+    @abstractmethod
+    def upload_contents(self, path: Path) -> None:
+        pass
+
+    def run(self, path: Path, file_data: FileData, **kwargs: Any) -> None:
+        self.upload_contents(path=path)

unstructured_ingest/v2/processes/connectors/sql/sqlite.py

@@ -0,0 +1,172 @@
+import json
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+import numpy as np
+import pandas as pd
+from pydantic import Field, Secret, model_validator
+
+from unstructured_ingest.v2.interfaces import FileData
+from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.v2.processes.connector_registry import DestinationRegistryEntry
+from unstructured_ingest.v2.processes.connectors.sql.sql import (
+    _DATE_COLUMNS,
+    SQLAccessConfig,
+    SQLConnectionConfig,
+    SQLDownloader,
+    SQLDownloaderConfig,
+    SQLIndexer,
+    SQLIndexerConfig,
+    SQLUploader,
+    SQLUploaderConfig,
+    SQLUploadStager,
+    SQLUploadStagerConfig,
+    parse_date_string,
+)
+
+if TYPE_CHECKING:
+    from sqlite3 import Connection as SqliteConnection
+
+CONNECTOR_TYPE = "sqlite"
+
+
+class SQLiteAccessConfig(SQLAccessConfig):
+    pass
+
+
+class SQLiteConnectionConfig(SQLConnectionConfig):
+    access_config: Secret[SQLiteAccessConfig] = Field(
+        default=SQLiteAccessConfig(), validate_default=True
+    )
+    database_path: Path = Field(
+        description="Path to the .db file.",
+    )
+
+    @model_validator(mode="after")
+    def check_database_path(self) -> "SQLiteConnectionConfig":
+        if not self.database_path.exists():
+            raise ValueError(f"{self.database_path} does not exist")
+        if not self.database_path.is_file():
+            raise ValueError(f"{self.database_path} is not a valid file")
+        return self
+
+    def get_connection(self) -> "SqliteConnection":
+        from sqlite3 import connect
+
+        return connect(database=self.database_path)
+
+
+class SQLiteIndexerConfig(SQLIndexerConfig):
+    pass
+
+
+@dataclass
+class SQLiteIndexer(SQLIndexer):
+    connection_config: SQLConnectionConfig
+    index_config: SQLIndexerConfig
+    connector_type: str = CONNECTOR_TYPE
+
+    def _get_doc_ids(self) -> list[str]:
+        with self.connection_config.get_connection() as sqlite_connection:
+            cursor = sqlite_connection.cursor()
+            cursor.execute(
+                f"SELECT {self.index_config.id_column} FROM {self.index_config.table_name}"
+            )
+            results = cursor.fetchall()
+            ids = [result[0] for result in results]
+            return ids
+
+
+class SQLiteDownloaderConfig(SQLDownloaderConfig):
+    pass
+
+
+@dataclass
+class SQLiteDownloader(SQLDownloader):
+    connection_config: SQLConnectionConfig
+    download_config: SQLDownloaderConfig
+    connector_type: str = CONNECTOR_TYPE
+
+    def query_db(self, file_data: FileData) -> tuple[list[tuple], list[str]]:
+        table_name = file_data.additional_metadata["table_name"]
+        id_column = file_data.additional_metadata["id_column"]
+        ids = file_data.additional_metadata["ids"]
+        with self.connection_config.get_connection() as sqlite_connection:
+            cursor = sqlite_connection.cursor()
+            fields = ",".join(self.download_config.fields) if self.download_config.fields else "*"
+            query = "SELECT {fields} FROM {table_name} WHERE {id_column} in ({ids})".format(
+                fields=fields,
+                table_name=table_name,
+                id_column=id_column,
+                ids=",".join([str(i) for i in ids]),
+            )
+            logger.debug(f"running query: {query}")
+            cursor.execute(query)
+            rows = cursor.fetchall()
+            columns = [col[0] for col in cursor.description]
+            return rows, columns
+
+
+class SQLiteUploadStagerConfig(SQLUploadStagerConfig):
+    pass
+
+
+class SQLiteUploadStager(SQLUploadStager):
+    upload_stager_config: SQLiteUploadStagerConfig
+
+
+class SQLiteUploaderConfig(SQLUploaderConfig):
+    pass
+
+
+@dataclass
+class SQLiteUploader(SQLUploader):
+    upload_config: SQLiteUploaderConfig = field(default_factory=SQLiteUploaderConfig)
+    connection_config: SQLiteConnectionConfig
+    connector_type: str = CONNECTOR_TYPE
+
+    def prepare_data(
+        self, columns: list[str], data: tuple[tuple[Any, ...], ...]
+    ) -> list[tuple[Any, ...]]:
+        output = []
+        for row in data:
+            parsed = []
+            for column_name, value in zip(columns, row):
+                if isinstance(value, (list, dict)):
+                    value = json.dumps(value)
+                if column_name in _DATE_COLUMNS:
+                    if value is None:
+                        parsed.append(None)
+                    else:
+                        parsed.append(parse_date_string(value))
+                else:
+                    parsed.append(value)
+            output.append(tuple(parsed))
+        return output
+
+    def upload_contents(self, path: Path) -> None:
+        df = pd.read_json(path, orient="records", lines=True)
+        logger.debug(f"uploading {len(df)} entries to {self.connection_config.database_path} ")
+        df.replace({np.nan: None}, inplace=True)
+
+        columns = tuple(df.columns)
+        stmt = f"INSERT INTO {self.upload_config.table_name} ({','.join(columns)}) \
+                VALUES({','.join(['?' for x in columns])})"  # noqa E501
+
+        for rows in pd.read_json(
+            path, orient="records", lines=True, chunksize=self.upload_config.batch_size
+        ):
+            with self.connection_config.get_connection() as conn:
+                values = self.prepare_data(columns, tuple(rows.itertuples(index=False, name=None)))
+                conn.executemany(stmt, values)
+                conn.commit()
+
+
+sqlite_destination_entry = DestinationRegistryEntry(
+    connection_config=SQLiteConnectionConfig,
+    uploader=SQLiteUploader,
+    uploader_config=SQLiteUploaderConfig,
+    upload_stager=SQLiteUploadStager,
+    upload_stager_config=SQLiteUploadStagerConfig,
+)

unstructured_ingest/v2/processes/embedder.py

@@ -22,6 +22,7 @@ class EmbedderConfig(BaseModel):
             "voyageai",
             "octoai",
             "mixedbread-ai",
+            "togetherai",
         ]
     ] = Field(default=None, description="Type of the embedding class to be used.")
     embedding_api_key: Optional[SecretStr] = Field(
@@ -107,6 +108,16 @@ class EmbedderConfig(BaseModel):
             config=MixedbreadAIEmbeddingConfig.model_validate(embedding_kwargs)
         )
 
+    def get_togetherai_embedder(self, embedding_kwargs: dict) -> "BaseEmbeddingEncoder":
+        from unstructured_ingest.embed.togetherai import (
+            TogetherAIEmbeddingConfig,
+            TogetherAIEmbeddingEncoder,
+        )
+
+        return TogetherAIEmbeddingEncoder(
+            config=TogetherAIEmbeddingConfig.model_validate(embedding_kwargs)
+        )
+
     def get_embedder(self) -> "BaseEmbeddingEncoder":
         kwargs: dict[str, Any] = {}
         if self.embedding_api_key:
@@ -133,6 +144,8 @@ class EmbedderConfig(BaseModel):
             return self.get_voyageai_embedder(embedding_kwargs=kwargs)
         if self.embedding_provider == "mixedbread-ai":
             return self.get_mixedbread_embedder(embedding_kwargs=kwargs)
+        if self.embedding_provider == "togetherai":
+            return self.get_togetherai_embedder(embedding_kwargs=kwargs)
 
         raise ValueError(f"{self.embedding_provider} not a recognized encoder")
 

unstructured_ingest/v2/processes/partitioner.py

@@ -55,7 +55,7 @@ class PartitionerConfig(BaseModel):
         "fields if they exist and drop all other fields. ",
     )
     partition_endpoint: Optional[str] = Field(
-        default="https://api.unstructured.io/general/v0/general",
+        default="https://api.unstructuredapp.io/general/v0/general",
         description="If partitioning via api, use the following host.",
     )
     partition_by_api: bool = Field(
@@ -153,6 +153,7 @@ class Partitioner(BaseProcess, ABC):
     async def partition_via_api(
         self, filename: Path, metadata: Optional[dict] = None, **kwargs
     ) -> list[dict]:
+        metadata = metadata or {}
         logger.debug(f"partitioning file {filename} with metadata: {metadata}")
 
         elements = await call_api(

{unstructured_ingest-0.0.25.dist-info → unstructured_ingest-0.1.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unstructured-ingest
-Version: 0.0.25
+Version: 0.1.1
 Summary: A library that prepares raw documents for downstream ML tasks.
 Home-page: https://github.com/Unstructured-IO/unstructured-ingest
 Author: Unstructured Technologies
@@ -22,13 +22,13 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.9.0,<3.13
 Description-Content-Type: text/markdown
 License-File: LICENSE.md
+Requires-Dist: python-dateutil
+Requires-Dist: pandas
 Requires-Dist: pydantic>=2.7
 Requires-Dist: dataclasses-json
 Requires-Dist: opentelemetry-sdk
-Requires-Dist: python-dateutil
-Requires-Dist: pandas
-Requires-Dist: tqdm
 Requires-Dist: click
+Requires-Dist: tqdm
 Provides-Extra: airtable
 Requires-Dist: pyairtable; extra == "airtable"
 Provides-Extra: astradb
@@ -41,11 +41,11 @@ Requires-Dist: azure-search-documents; extra == "azure-cognitive-search"
 Provides-Extra: bedrock
 Requires-Dist: boto3; extra == "bedrock"
 Provides-Extra: biomed
-Requires-Dist: requests; extra == "biomed"
 Requires-Dist: bs4; extra == "biomed"
+Requires-Dist: requests; extra == "biomed"
 Provides-Extra: box
-Requires-Dist: fsspec; extra == "box"
 Requires-Dist: boxfs; extra == "box"
+Requires-Dist: fsspec; extra == "box"
 Provides-Extra: chroma
 Requires-Dist: chromadb; extra == "chroma"
 Provides-Extra: clarifai
@@ -69,8 +69,8 @@ Requires-Dist: unstructured[docx]; extra == "doc"
 Provides-Extra: docx
 Requires-Dist: unstructured[docx]; extra == "docx"
 Provides-Extra: dropbox
-Requires-Dist: dropboxdrivefs; extra == "dropbox"
 Requires-Dist: fsspec; extra == "dropbox"
+Requires-Dist: dropboxdrivefs; extra == "dropbox"
 Provides-Extra: elasticsearch
 Requires-Dist: elasticsearch[async]; extra == "elasticsearch"
 Provides-Extra: embed-huggingface
@@ -88,11 +88,11 @@ Provides-Extra: epub
 Requires-Dist: unstructured[epub]; extra == "epub"
 Provides-Extra: gcs
 Requires-Dist: gcsfs; extra == "gcs"
-Requires-Dist: fsspec; extra == "gcs"
 Requires-Dist: bs4; extra == "gcs"
+Requires-Dist: fsspec; extra == "gcs"
 Provides-Extra: github
-Requires-Dist: requests; extra == "github"
 Requires-Dist: pygithub>1.58.0; extra == "github"
+Requires-Dist: requests; extra == "github"
 Provides-Extra: gitlab
 Requires-Dist: python-gitlab; extra == "gitlab"
 Provides-Extra: google-drive
@@ -105,7 +105,7 @@ Requires-Dist: atlassian-python-api; extra == "jira"
 Provides-Extra: kafka
 Requires-Dist: confluent-kafka; extra == "kafka"
 Provides-Extra: kdbai
-Requires-Dist: kdbai-client; extra == "kdbai"
+Requires-Dist: kdbai-client>=1.4.0; extra == "kdbai"
 Provides-Extra: md
 Requires-Dist: unstructured[md]; extra == "md"
 Provides-Extra: milvus
@@ -116,15 +116,15 @@ Provides-Extra: msg
 Requires-Dist: unstructured[msg]; extra == "msg"
 Provides-Extra: notion
 Requires-Dist: notion-client; extra == "notion"
-Requires-Dist: httpx; extra == "notion"
 Requires-Dist: htmlBuilder; extra == "notion"
 Requires-Dist: backoff; extra == "notion"
+Requires-Dist: httpx; extra == "notion"
 Provides-Extra: odt
 Requires-Dist: unstructured[odt]; extra == "odt"
 Provides-Extra: onedrive
 Requires-Dist: Office365-REST-Python-Client; extra == "onedrive"
-Requires-Dist: msal; extra == "onedrive"
 Requires-Dist: bs4; extra == "onedrive"
+Requires-Dist: msal; extra == "onedrive"
 Provides-Extra: openai
 Requires-Dist: openai; extra == "openai"
 Requires-Dist: tiktoken; extra == "openai"
@@ -156,13 +156,13 @@ Requires-Dist: unstructured[rst]; extra == "rst"
 Provides-Extra: rtf
 Requires-Dist: unstructured[rtf]; extra == "rtf"
 Provides-Extra: s3
-Requires-Dist: s3fs; extra == "s3"
 Requires-Dist: fsspec; extra == "s3"
+Requires-Dist: s3fs; extra == "s3"
 Provides-Extra: salesforce
 Requires-Dist: simple-salesforce; extra == "salesforce"
 Provides-Extra: sftp
-Requires-Dist: paramiko; extra == "sftp"
 Requires-Dist: fsspec; extra == "sftp"
+Requires-Dist: paramiko; extra == "sftp"
 Provides-Extra: sharepoint
 Requires-Dist: Office365-REST-Python-Client; extra == "sharepoint"
 Requires-Dist: msal; extra == "sharepoint"
@@ -170,6 +170,8 @@ Provides-Extra: singlestore
 Requires-Dist: singlestoredb; extra == "singlestore"
 Provides-Extra: slack
 Requires-Dist: slack-sdk; extra == "slack"
+Provides-Extra: togetherai
+Requires-Dist: together; extra == "togetherai"
 Provides-Extra: tsv
 Requires-Dist: unstructured[tsv]; extra == "tsv"
 Provides-Extra: vectara