unstructured-ingest 0.3.7__py3-none-any.whl → 0.3.9__py3-none-any.whl

unstructured_ingest/v2/processes/connectors/qdrant/qdrant.py

@@ -1,10 +1,9 @@
 import asyncio
 import json
 from abc import ABC, abstractmethod
-from contextlib import asynccontextmanager
+from contextlib import asynccontextmanager, contextmanager
 from dataclasses import dataclass, field
-from pathlib import Path
-from typing import TYPE_CHECKING, Any, AsyncGenerator, Optional
+from typing import TYPE_CHECKING, Any, AsyncGenerator, Generator, Optional
 
 from pydantic import Field, Secret
 
@@ -24,7 +23,7 @@ from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.utils import get_enhanced_element_id
 
 if TYPE_CHECKING:
-    from qdrant_client import AsyncQdrantClient
+    from qdrant_client import AsyncQdrantClient, QdrantClient
 
 
 class QdrantAccessConfig(AccessConfig, ABC):
@@ -42,8 +41,8 @@ class QdrantConnectionConfig(ConnectionConfig, ABC):
 
     @requires_dependencies(["qdrant_client"], extras="qdrant")
     @asynccontextmanager
-    async def get_client(self) -> AsyncGenerator["AsyncQdrantClient", None]:
-        from qdrant_client.async_qdrant_client import AsyncQdrantClient
+    async def get_async_client(self) -> AsyncGenerator["AsyncQdrantClient", None]:
+        from qdrant_client import AsyncQdrantClient
 
         client_kwargs = self.get_client_kwargs()
         client = AsyncQdrantClient(**client_kwargs)
@@ -52,6 +51,18 @@ class QdrantConnectionConfig(ConnectionConfig, ABC):
         finally:
             await client.close()
 
+    @requires_dependencies(["qdrant_client"], extras="qdrant")
+    @contextmanager
+    def get_client(self) -> Generator["QdrantClient", None, None]:
+        from qdrant_client import QdrantClient
+
+        client_kwargs = self.get_client_kwargs()
+        client = QdrantClient(**client_kwargs)
+        try:
+            yield client
+        finally:
+            client.close()
+
 
 class QdrantUploadStagerConfig(UploadStagerConfig):
     pass
@@ -63,9 +74,9 @@ class QdrantUploadStager(UploadStager, ABC):
         default_factory=lambda: QdrantUploadStagerConfig()
     )
 
-    @staticmethod
-    def conform_dict(data: dict, file_data: FileData) -> dict:
+    def conform_dict(self, element_dict: dict, file_data: FileData) -> dict:
         """Prepares dictionary in the format that Chroma requires"""
+        data = element_dict.copy()
         return {
             "id": get_enhanced_element_id(element_dict=data, file_data=file_data),
             "vector": data.pop("embeddings", {}),
@@ -80,26 +91,6 @@ class QdrantUploadStager(UploadStager, ABC):
             },
         }
 
-    def run(
-        self,
-        elements_filepath: Path,
-        file_data: FileData,
-        output_dir: Path,
-        output_filename: str,
-        **kwargs: Any,
-    ) -> Path:
-        with open(elements_filepath) as elements_file:
-            elements_contents = json.load(elements_file)
-
-        conformed_elements = [
-            self.conform_dict(data=element, file_data=file_data) for element in elements_contents
-        ]
-        output_path = Path(output_dir) / Path(f"{output_filename}.json")
-
-        with open(output_path, "w") as output_file:
-            json.dump(conformed_elements, output_file)
-        return output_path
-
 
 class QdrantUploaderConfig(UploaderConfig):
     collection_name: str = Field(description="Name of the collection.")
@@ -118,27 +109,27 @@ class QdrantUploader(Uploader, ABC):
 
     @DestinationConnectionError.wrap
     def precheck(self) -> None:
-        async def check_connection():
-            async with self.connection_config.get_client() as async_client:
-                await async_client.get_collections()
-
-        asyncio.run(check_connection())
+        with self.connection_config.get_client() as client:
+            collections_response = client.get_collections()
+            collection_names = [c.name for c in collections_response.collections]
+            if self.upload_config.collection_name not in collection_names:
+                raise DestinationConnectionError(
+                    "collection '{}' not found: {}".format(
+                        self.upload_config.collection_name, ", ".join(collection_names)
+                    )
+                )
 
     def is_async(self):
         return True
 
-    async def run_async(
+    async def run_data_async(
         self,
-        path: Path,
+        data: list[dict],
         file_data: FileData,
         **kwargs: Any,
     ) -> None:
-        with path.open("r") as file:
-            elements: list[dict] = json.load(file)
-
-        logger.debug("Loaded %i elements from %s", len(elements), path)
 
-        batches = list(batch_generator(elements, batch_size=self.upload_config.batch_size))
+        batches = list(batch_generator(data, batch_size=self.upload_config.batch_size))
         logger.debug(
             "Elements split into %i batches of size %i.",
             len(batches),
@@ -156,7 +147,7 @@ class QdrantUploader(Uploader, ABC):
                 len(points),
                 self.upload_config.collection_name,
             )
-            async with self.connection_config.get_client() as async_client:
+            async with self.connection_config.get_async_client() as async_client:
                 await async_client.upsert(
                     self.upload_config.collection_name, points=points, wait=True
                 )

unstructured_ingest/v2/processes/connectors/sql/sql.py

@@ -15,7 +15,7 @@ from dateutil import parser
 from pydantic import Field, Secret
 
 from unstructured_ingest.error import DestinationConnectionError, SourceConnectionError
-from unstructured_ingest.utils.data_prep import split_dataframe
+from unstructured_ingest.utils.data_prep import get_data_df, split_dataframe
 from unstructured_ingest.v2.constants import RECORD_ID_LABEL
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
@@ -238,27 +238,24 @@ class SQLUploadStagerConfig(UploadStagerConfig):
 class SQLUploadStager(UploadStager):
     upload_stager_config: SQLUploadStagerConfig = field(default_factory=SQLUploadStagerConfig)
 
-    @staticmethod
-    def conform_dict(data: dict, file_data: FileData) -> pd.DataFrame:
-        working_data = data.copy()
-        output = []
-        for element in working_data:
-            metadata: dict[str, Any] = element.pop("metadata", {})
-            data_source = metadata.pop("data_source", {})
-            coordinates = metadata.pop("coordinates", {})
+    def conform_dict(self, element_dict: dict, file_data: FileData) -> dict:
+        data = element_dict.copy()
+        metadata: dict[str, Any] = data.pop("metadata", {})
+        data_source = metadata.pop("data_source", {})
+        coordinates = metadata.pop("coordinates", {})
 
-            element.update(metadata)
-            element.update(data_source)
-            element.update(coordinates)
+        data.update(metadata)
+        data.update(data_source)
+        data.update(coordinates)
 
-            element["id"] = get_enhanced_element_id(element_dict=element, file_data=file_data)
+        data["id"] = get_enhanced_element_id(element_dict=data, file_data=file_data)
 
-            # remove extraneous, not supported columns
-            element = {k: v for k, v in element.items() if k in _COLUMNS}
-            element[RECORD_ID_LABEL] = file_data.identifier
-            output.append(element)
+        # remove extraneous, not supported columns
+        element = {k: v for k, v in data.items() if k in _COLUMNS}
+        element[RECORD_ID_LABEL] = file_data.identifier
+        return element
 
-        df = pd.DataFrame.from_dict(output)
+    def conform_dataframe(self, df: pd.DataFrame) -> pd.DataFrame:
         for column in filter(lambda x: x in df.columns, _DATE_COLUMNS):
             df[column] = df[column].apply(parse_date_string)
         for column in filter(
@@ -283,19 +280,19 @@ class SQLUploadStager(UploadStager):
         output_filename: str,
         **kwargs: Any,
     ) -> Path:
-        with open(elements_filepath) as elements_file:
-            elements_contents: list[dict] = json.load(elements_file)
+        elements_contents = self.get_data(elements_filepath=elements_filepath)
 
-        df = self.conform_dict(data=elements_contents, file_data=file_data)
-        if Path(output_filename).suffix != ".json":
-            output_filename = f"{output_filename}.json"
-        else:
-            output_filename = f"{Path(output_filename).stem}.json"
-        output_path = Path(output_dir) / Path(f"{output_filename}")
-        output_path.parent.mkdir(parents=True, exist_ok=True)
+        df = pd.DataFrame(
+            data=[
+                self.conform_dict(element_dict=element_dict, file_data=file_data)
+                for element_dict in elements_contents
+            ]
+        )
+        df = self.conform_dataframe(df=df)
 
-        with output_path.open("w") as output_file:
-            df.to_json(output_file, orient="records", lines=True)
+        output_path = self.get_output_path(output_filename=output_filename, output_dir=output_dir)
+
+        self.write_output(output_path=output_path, data=df.to_dict(orient="records"))
         return output_path
 
 
@@ -361,8 +358,15 @@ class SQLUploader(Uploader):
         for column in missing_columns:
             df[column] = pd.Series()
 
-    def upload_contents(self, path: Path) -> None:
-        df = pd.read_json(path, orient="records", lines=True)
+    def upload_dataframe(self, df: pd.DataFrame, file_data: FileData) -> None:
+        if self.can_delete():
+            self.delete_by_record_id(file_data=file_data)
+        else:
+            logger.warning(
+                f"table doesn't contain expected "
+                f"record id column "
+                f"{self.upload_config.record_id_key}, skipping delete"
+            )
         df.replace({np.nan: None}, inplace=True)
         self._fit_to_schema(df=df, columns=self.get_table_columns())
 
@@ -411,13 +415,10 @@ class SQLUploader(Uploader):
             rowcount = cursor.rowcount
             logger.info(f"deleted {rowcount} rows from table {self.upload_config.table_name}")
 
+    def run_data(self, data: list[dict], file_data: FileData, **kwargs: Any) -> None:
+        df = pd.DataFrame(data)
+        self.upload_dataframe(df=df, file_data=file_data)
+
     def run(self, path: Path, file_data: FileData, **kwargs: Any) -> None:
-        if self.can_delete():
-            self.delete_by_record_id(file_data=file_data)
-        else:
-            logger.warning(
-                f"table doesn't contain expected "
-                f"record id column "
-                f"{self.upload_config.record_id_key}, skipping delete"
-            )
-        self.upload_contents(path=path)
+        df = get_data_df(path=path)
+        self.upload_dataframe(df=df, file_data=file_data)

unstructured_ingest/v2/processes/connectors/weaviate/weaviate.py

@@ -3,7 +3,6 @@ from abc import ABC, abstractmethod
 from contextlib import contextmanager
 from dataclasses import dataclass, field
 from datetime import date, datetime
-from pathlib import Path
 from typing import TYPE_CHECKING, Any, Generator, Optional
 
 from dateutil import parser
@@ -74,11 +73,11 @@ class WeaviateUploadStager(UploadStager):
             logger.debug(f"date {date_string} string not a timestamp: {e}")
         return parser.parse(date_string)
 
-    @classmethod
-    def conform_dict(cls, data: dict, file_data: FileData) -> dict:
+    def conform_dict(self, element_dict: dict, file_data: FileData) -> dict:
         """
         Updates the element dictionary to conform to the Weaviate schema
         """
+        data = element_dict.copy()
         working_data = data.copy()
         # Dict as string formatting
         if (
@@ -111,7 +110,7 @@ class WeaviateUploadStager(UploadStager):
             .get("data_source", {})
             .get("date_created")
         ):
-            working_data["metadata"]["data_source"]["date_created"] = cls.parse_date_string(
+            working_data["metadata"]["data_source"]["date_created"] = self.parse_date_string(
                 date_created
             ).strftime(
                 "%Y-%m-%dT%H:%M:%S.%fZ",
@@ -122,7 +121,7 @@ class WeaviateUploadStager(UploadStager):
             .get("data_source", {})
             .get("date_modified")
         ):
-            working_data["metadata"]["data_source"]["date_modified"] = cls.parse_date_string(
+            working_data["metadata"]["data_source"]["date_modified"] = self.parse_date_string(
                 date_modified
             ).strftime(
                 "%Y-%m-%dT%H:%M:%S.%fZ",
@@ -133,14 +132,14 @@ class WeaviateUploadStager(UploadStager):
             .get("data_source", {})
             .get("date_processed")
         ):
-            working_data["metadata"]["data_source"]["date_processed"] = cls.parse_date_string(
+            working_data["metadata"]["data_source"]["date_processed"] = self.parse_date_string(
                 date_processed
             ).strftime(
                 "%Y-%m-%dT%H:%M:%S.%fZ",
             )
 
         if last_modified := working_data.get("metadata", {}).get("last_modified"):
-            working_data["metadata"]["last_modified"] = cls.parse_date_string(
+            working_data["metadata"]["last_modified"] = self.parse_date_string(
                 last_modified
             ).strftime(
                 "%Y-%m-%dT%H:%M:%S.%fZ",
@@ -159,25 +158,6 @@ class WeaviateUploadStager(UploadStager):
         working_data[RECORD_ID_LABEL] = file_data.identifier
         return working_data
 
-    def run(
-        self,
-        elements_filepath: Path,
-        file_data: FileData,
-        output_dir: Path,
-        output_filename: str,
-        **kwargs: Any,
-    ) -> Path:
-        with open(elements_filepath) as elements_file:
-            elements_contents = json.load(elements_file)
-        updated_elements = [
-            self.conform_dict(data=element, file_data=file_data) for element in elements_contents
-        ]
-        output_path = Path(output_dir) / Path(f"{output_filename}.json")
-        output_path.parent.mkdir(parents=True, exist_ok=True)
-        with open(output_path, "w") as output_file:
-            json.dump(updated_elements, output_file, indent=2)
-        return output_path
-
 
 class WeaviateUploaderConfig(UploaderConfig):
     collection: str = Field(description="The name of the collection this object belongs to")
@@ -268,18 +248,16 @@ class WeaviateUploader(Uploader, ABC):
             if not resp.failed and not resp.successful:
                 break
 
-    def run(self, path: Path, file_data: FileData, **kwargs: Any) -> None:
-        with path.open("r") as file:
-            elements_dict = json.load(file)
+    def run_data(self, data: list[dict], file_data: FileData, **kwargs: Any) -> None:
         logger.info(
-            f"writing {len(elements_dict)} objects to destination "
+            f"writing {len(data)} objects to destination "
             f"class {self.connection_config.access_config} "
         )
 
         with self.connection_config.get_client() as weaviate_client:
             self.delete_by_record_id(client=weaviate_client, file_data=file_data)
             with self.upload_config.get_batch_client(client=weaviate_client) as batch_client:
-                for e in elements_dict:
+                for e in data:
                     vector = e.pop("embeddings", None)
                     batch_client.add_object(
                         collection=self.upload_config.collection,

{unstructured_ingest-0.3.7.dist-info → unstructured_ingest-0.3.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unstructured-ingest
-Version: 0.3.7
+Version: 0.3.9
 Summary: A library that prepares raw documents for downstream ML tasks.
 Home-page: https://github.com/Unstructured-IO/unstructured-ingest
 Author: Unstructured Technologies
@@ -22,13 +22,14 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.9.0,<3.13
 Description-Content-Type: text/markdown
 License-File: LICENSE.md
-Requires-Dist: pandas
-Requires-Dist: tqdm
+Requires-Dist: click
 Requires-Dist: pydantic>=2.7
-Requires-Dist: opentelemetry-sdk
+Requires-Dist: pandas
 Requires-Dist: dataclasses-json
-Requires-Dist: click
 Requires-Dist: python-dateutil
+Requires-Dist: tqdm
+Requires-Dist: ndjson
+Requires-Dist: opentelemetry-sdk
 Provides-Extra: airtable
 Requires-Dist: pyairtable; extra == "airtable"
 Provides-Extra: astradb
@@ -41,8 +42,8 @@ Requires-Dist: azure-search-documents; extra == "azure-ai-search"
 Provides-Extra: bedrock
 Requires-Dist: boto3; extra == "bedrock"
 Provides-Extra: biomed
-Requires-Dist: requests; extra == "biomed"
 Requires-Dist: bs4; extra == "biomed"
+Requires-Dist: requests; extra == "biomed"
 Provides-Extra: box
 Requires-Dist: fsspec; extra == "box"
 Requires-Dist: boxfs; extra == "box"
@@ -69,8 +70,8 @@ Requires-Dist: unstructured[docx]; extra == "doc"
 Provides-Extra: docx
 Requires-Dist: unstructured[docx]; extra == "docx"
 Provides-Extra: dropbox
-Requires-Dist: dropboxdrivefs; extra == "dropbox"
 Requires-Dist: fsspec; extra == "dropbox"
+Requires-Dist: dropboxdrivefs; extra == "dropbox"
 Provides-Extra: duckdb
 Requires-Dist: duckdb; extra == "duckdb"
 Provides-Extra: elasticsearch
@@ -80,8 +81,8 @@ Requires-Dist: sentence-transformers; extra == "embed-huggingface"
 Provides-Extra: embed-mixedbreadai
 Requires-Dist: mixedbread-ai; extra == "embed-mixedbreadai"
 Provides-Extra: embed-octoai
-Requires-Dist: tiktoken; extra == "embed-octoai"
 Requires-Dist: openai; extra == "embed-octoai"
+Requires-Dist: tiktoken; extra == "embed-octoai"
 Provides-Extra: embed-vertexai
 Requires-Dist: vertexai; extra == "embed-vertexai"
 Provides-Extra: embed-voyageai
@@ -89,19 +90,19 @@ Requires-Dist: voyageai; extra == "embed-voyageai"
 Provides-Extra: epub
 Requires-Dist: unstructured[epub]; extra == "epub"
 Provides-Extra: gcs
-Requires-Dist: gcsfs; extra == "gcs"
 Requires-Dist: fsspec; extra == "gcs"
 Requires-Dist: bs4; extra == "gcs"
+Requires-Dist: gcsfs; extra == "gcs"
 Provides-Extra: github
-Requires-Dist: requests; extra == "github"
 Requires-Dist: pygithub>1.58.0; extra == "github"
+Requires-Dist: requests; extra == "github"
 Provides-Extra: gitlab
 Requires-Dist: python-gitlab; extra == "gitlab"
 Provides-Extra: google-drive
 Requires-Dist: google-api-python-client; extra == "google-drive"
 Provides-Extra: hubspot
-Requires-Dist: hubspot-api-client; extra == "hubspot"
 Requires-Dist: urllib3; extra == "hubspot"
+Requires-Dist: hubspot-api-client; extra == "hubspot"
 Provides-Extra: jira
 Requires-Dist: atlassian-python-api; extra == "jira"
 Provides-Extra: kafka
@@ -118,11 +119,14 @@ Provides-Extra: mongodb
 Requires-Dist: pymongo; extra == "mongodb"
 Provides-Extra: msg
 Requires-Dist: unstructured[msg]; extra == "msg"
+Provides-Extra: neo4j
+Requires-Dist: neo4j; extra == "neo4j"
+Requires-Dist: cymple; extra == "neo4j"
 Provides-Extra: notion
-Requires-Dist: notion-client; extra == "notion"
-Requires-Dist: backoff; extra == "notion"
 Requires-Dist: htmlBuilder; extra == "notion"
 Requires-Dist: httpx; extra == "notion"
+Requires-Dist: backoff; extra == "notion"
+Requires-Dist: notion-client; extra == "notion"
 Provides-Extra: odt
 Requires-Dist: unstructured[odt]; extra == "odt"
 Provides-Extra: onedrive
@@ -130,8 +134,8 @@ Requires-Dist: Office365-REST-Python-Client; extra == "onedrive"
 Requires-Dist: msal; extra == "onedrive"
 Requires-Dist: bs4; extra == "onedrive"
 Provides-Extra: openai
-Requires-Dist: tiktoken; extra == "openai"
 Requires-Dist: openai; extra == "openai"
+Requires-Dist: tiktoken; extra == "openai"
 Provides-Extra: opensearch
 Requires-Dist: opensearch-py; extra == "opensearch"
 Provides-Extra: org
@@ -160,13 +164,13 @@ Requires-Dist: unstructured[rst]; extra == "rst"
 Provides-Extra: rtf
 Requires-Dist: unstructured[rtf]; extra == "rtf"
 Provides-Extra: s3
-Requires-Dist: s3fs; extra == "s3"
 Requires-Dist: fsspec; extra == "s3"
+Requires-Dist: s3fs; extra == "s3"
 Provides-Extra: salesforce
 Requires-Dist: simple-salesforce; extra == "salesforce"
 Provides-Extra: sftp
-Requires-Dist: paramiko; extra == "sftp"
 Requires-Dist: fsspec; extra == "sftp"
+Requires-Dist: paramiko; extra == "sftp"
 Provides-Extra: sharepoint
 Requires-Dist: Office365-REST-Python-Client; extra == "sharepoint"
 Requires-Dist: msal; extra == "sharepoint"
@@ -175,8 +179,8 @@ Requires-Dist: singlestoredb; extra == "singlestore"
 Provides-Extra: slack
 Requires-Dist: slack-sdk[optional]; extra == "slack"
 Provides-Extra: snowflake
-Requires-Dist: psycopg2-binary; extra == "snowflake"
 Requires-Dist: snowflake-connector-python; extra == "snowflake"
+Requires-Dist: psycopg2-binary; extra == "snowflake"
 Provides-Extra: togetherai
 Requires-Dist: together; extra == "togetherai"
 Provides-Extra: tsv