unstructured-ingest 0.0.0__tar.gz → 0.0.1__tar.gz

{unstructured-ingest-0.0.0 → unstructured-ingest-0.0.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unstructured-ingest
-Version: 0.0.0
+Version: 0.0.1
 Summary: A library that prepares raw documents for downstream ML tasks.
 Home-page: https://github.com/Unstructured-IO/unstructured-ingest
 Author: Unstructured Technologies
@@ -56,6 +56,7 @@ Provides-Extra: google-drive
 Provides-Extra: hubspot
 Provides-Extra: jira
 Provides-Extra: kafka
+Provides-Extra: milvus
 Provides-Extra: mongodb
 Provides-Extra: notion
 Provides-Extra: onedrive

{unstructured-ingest-0.0.0 → unstructured-ingest-0.0.1}/setup.py

@@ -102,6 +102,7 @@ connectors_reqs = {
     "hubspot": load_requirements("requirements/connectors/hubspot.in"),
     "jira": load_requirements("requirements/connectors/jira.in"),
     "kafka": load_requirements("requirements/connectors/kafka.in"),
+    "milvus": load_requirements("requirements/connectors/milvus.in"),
     "mongodb": load_requirements("requirements/connectors/mongodb.in"),
     "notion": load_requirements("requirements/connectors/notion.in"),
     "onedrive": load_requirements("requirements/connectors/onedrive.in"),

unstructured-ingest-0.0.1/unstructured_ingest/__version__.py

@@ -0,0 +1 @@
+__version__ = "0.0.1"  # pragma: no cover

{unstructured-ingest-0.0.0 → unstructured-ingest-0.0.1}/unstructured_ingest/connector/notion/helpers.py

@@ -5,7 +5,6 @@ from typing import List, Optional, Tuple
 from urllib.parse import urlparse
 from uuid import UUID
 
-import unstructured.ingest.connector.notion.types.blocks as notion_blocks
 from htmlBuilder.attributes import Style, Type
 from htmlBuilder.tags import (
     Body,
@@ -23,6 +22,7 @@ from htmlBuilder.tags import (
 )
 from notion_client.errors import APIResponseError
 
+import unstructured_ingest.connector.notion.types.blocks as notion_blocks
 from unstructured_ingest.connector.notion.client import Client
 from unstructured_ingest.connector.notion.interfaces import BlockBase
 from unstructured_ingest.connector.notion.types.block import Block

{unstructured-ingest-0.0.0 → unstructured-ingest-0.0.1}/unstructured_ingest/logger.py

@@ -3,7 +3,7 @@ import json
 import logging
 import typing as t
 
-logger = logging.getLogger("unstructured.ingest")
+logger = logging.getLogger("unstructured_ingest")
 
 
 def default_is_data_sensitive(k: str, v: t.Any) -> bool:
@@ -119,7 +119,7 @@ def ingest_log_streaming_init(level: int) -> None:
 
 def make_default_logger(level: int) -> logging.Logger:
     """Return a custom logger."""
-    logger = logging.getLogger("unstructured.ingest")
+    logger = logging.getLogger("unstructured_ingest")
     handler = logging.StreamHandler()
     handler.name = "ingest_log_handler"
     formatter = SensitiveFormatter("%(asctime)s %(processName)-10s %(levelname)-8s %(message)s")

{unstructured-ingest-0.0.0 → unstructured-ingest-0.0.1}/unstructured_ingest/v2/cli/cmds/__init__.py

@@ -15,6 +15,7 @@ from .fsspec.s3 import s3_dest_cmd, s3_src_cmd
 from .fsspec.sftp import sftp_dest_cmd, sftp_src_cmd
 from .google_drive import google_drive_src_cmd
 from .local import local_dest_cmd, local_src_cmd
+from .milvus import milvus_dest_cmd
 from .mongodb import mongodb_dest_cmd
 from .onedrive import onedrive_drive_src_cmd
 from .opensearch import opensearch_dest_cmd, opensearch_src_cmd
@@ -60,6 +61,7 @@ dest_cmds = [
     elasticsearch_dest_cmd,
     gcs_dest_cmd,
     local_dest_cmd,
+    milvus_dest_cmd,
     opensearch_dest_cmd,
     pinecone_dest_cmd,
     s3_dest_cmd,

unstructured-ingest-0.0.1/unstructured_ingest/v2/cli/cmds/milvus.py

@@ -0,0 +1,72 @@
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.v2.cli.base import DestCmd
+from unstructured_ingest.v2.cli.interfaces import CliConfig
+from unstructured_ingest.v2.processes.connectors.milvus import CONNECTOR_TYPE
+
+
+@dataclass
+class MilvusCliConnectionConfig(CliConfig):
+    @staticmethod
+    def get_cli_options() -> list[click.Option]:
+        options = [
+            click.Option(
+                ["--uri"],
+                required=False,
+                type=str,
+                default=None,
+                help="Milvus uri, eg 'http://localhost:19530",
+            ),
+            click.Option(
+                ["--user"],
+                required=False,
+                type=str,
+                default=None,
+                help="Milvus user",
+            ),
+            click.Option(
+                ["--password"],
+                required=False,
+                type=str,
+                default=None,
+                help="Milvus password",
+            ),
+            click.Option(
+                ["--db-name"],
+                required=False,
+                type=str,
+                default=None,
+                help="Milvus database name",
+            ),
+        ]
+        return options
+
+
+@dataclass
+class MilvusCliUploaderConfig(CliConfig):
+    @staticmethod
+    def get_cli_options() -> list[click.Option]:
+        options = [
+            click.Option(
+                ["--collection-name"],
+                required=True,
+                type=str,
+                help="Milvus collections to write to",
+            ),
+            click.Option(
+                ["--num-of-processes"],
+                type=click.IntRange(min=1),
+                default=4,
+                help="number of processes to use when writing to support parallel writes",
+            ),
+        ]
+        return options
+
+
+milvus_dest_cmd = DestCmd(
+    cmd_name=CONNECTOR_TYPE,
+    connection_config=MilvusCliConnectionConfig,
+    uploader_config=MilvusCliUploaderConfig,
+)

{unstructured-ingest-0.0.0 → unstructured-ingest-0.0.1}/unstructured_ingest/v2/logger.py

@@ -5,7 +5,7 @@ from logging import Formatter, Logger, StreamHandler, getLevelName, getLogger
 from typing import Any, Callable
 
 log_level = os.getenv("INGEST_LOG_LEVEL", "INFO")
-LOGGER_NAME = "unstructured.ingest.v2"
+LOGGER_NAME = "unstructured_ingest.v2"
 
 
 def default_is_data_sensitive(k: str, v: Any) -> bool:

{unstructured-ingest-0.0.0 → unstructured-ingest-0.0.1}/unstructured_ingest/v2/pipeline/interfaces.py

@@ -146,6 +146,8 @@ class PipelineStep(ABC):
             logger.error(f"Exception raised while running {self.identifier}", exc_info=e)
             if "file_data_path" in kwargs:
                 self.context.status[kwargs["file_data_path"]] = {self.identifier: str(e)}
+            else:
+                self.context.status[self.identifier] = {"step_error": str(e)}
             if self.context.raise_on_error:
                 raise e
             return None
@@ -158,6 +160,8 @@ class PipelineStep(ABC):
             logger.error(f"Exception raised while running {self.identifier}", exc_info=e)
             if "file_data_path" in kwargs:
                 self.context.status[kwargs["file_data_path"]] = {self.identifier: str(e)}
+            else:
+                self.context.status[self.identifier] = {"step_error": str(e)}
             if self.context.raise_on_error:
                 raise e
             return None

{unstructured-ingest-0.0.0 → unstructured-ingest-0.0.1}/unstructured_ingest/v2/processes/connectors/__init__.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
-import unstructured.ingest.v2.processes.connectors.fsspec  # noqa: F401
-
+import unstructured_ingest.v2.processes.connectors.fsspec  # noqa: F401
 from unstructured_ingest.v2.processes.connector_registry import (
     add_destination_entry,
     add_source_entry,
@@ -19,6 +18,8 @@ from .google_drive import CONNECTOR_TYPE as GOOGLE_DRIVE_CONNECTOR_TYPE
 from .google_drive import google_drive_source_entry
 from .local import CONNECTOR_TYPE as LOCAL_CONNECTOR_TYPE
 from .local import local_destination_entry, local_source_entry
+from .milvus import CONNECTOR_TYPE as MILVUS_CONNECTOR_TYPE
+from .milvus import milvus_destination_entry
 from .mongodb import CONNECTOR_TYPE as MONGODB_CONNECTOR_TYPE
 from .mongodb import mongodb_destination_entry
 from .onedrive import CONNECTOR_TYPE as ONEDRIVE_CONNECTOR_TYPE
@@ -75,3 +76,4 @@ add_source_entry(source_type=SHAREPOINT_CONNECTOR_TYPE, entry=sharepoint_source_
 add_destination_entry(
     destination_type=SINGLESTORE_CONNECTOR_TYPE, entry=singlestore_destination_entry
 )
+add_destination_entry(destination_type=MILVUS_CONNECTOR_TYPE, entry=milvus_destination_entry)

unstructured-ingest-0.0.1/unstructured_ingest/v2/processes/connectors/milvus.py

@@ -0,0 +1,200 @@
+import json
+import multiprocessing as mp
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Optional, Union
+
+import pandas as pd
+from dateutil import parser
+
+from unstructured_ingest.enhanced_dataclass import enhanced_field
+from unstructured_ingest.error import WriteError
+from unstructured_ingest.utils.data_prep import flatten_dict
+from unstructured_ingest.utils.dep_check import requires_dependencies
+from unstructured_ingest.v2.interfaces import (
+    AccessConfig,
+    ConnectionConfig,
+    FileData,
+    UploadContent,
+    Uploader,
+    UploaderConfig,
+    UploadStager,
+    UploadStagerConfig,
+)
+from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.v2.processes.connector_registry import (
+    DestinationRegistryEntry,
+)
+
+if TYPE_CHECKING:
+    from pymilvus import MilvusClient
+
+CONNECTOR_TYPE = "milvus"
+
+
+@dataclass
+class MilvusAccessConfig(AccessConfig):
+    password: Optional[str] = None
+    token: Optional[str] = None
+
+
+@dataclass
+class MilvusConnectionConfig(ConnectionConfig):
+    access_config: MilvusAccessConfig = enhanced_field(
+        sensitive=True, default_factory=lambda: MilvusAccessConfig()
+    )
+    uri: Optional[str] = None
+    user: Optional[str] = None
+    db_name: Optional[str] = None
+
+    def get_connection_kwargs(self) -> dict[str, Any]:
+        access_config_dict = self.access_config.to_dict()
+        connection_config_dict = self.to_dict()
+        connection_config_dict.pop("access_config", None)
+        connection_config_dict.update(access_config_dict)
+        # Drop any that were not set explicitly
+        connection_config_dict = {k: v for k, v in connection_config_dict.items() if v is not None}
+        return connection_config_dict
+
+    @requires_dependencies(["pymilvus"], extras="milvus")
+    def get_client(self) -> "MilvusClient":
+        from pymilvus import MilvusClient
+
+        return MilvusClient(**self.get_connection_kwargs())
+
+
+@dataclass
+class MilvusUploadStagerConfig(UploadStagerConfig):
+    pass
+
+
+@dataclass
+class MilvusUploadStager(UploadStager):
+    upload_stager_config: MilvusUploadStagerConfig = field(
+        default_factory=lambda: MilvusUploadStagerConfig()
+    )
+
+    @staticmethod
+    def parse_date_string(date_string: str) -> float:
+        try:
+            timestamp = float(date_string)
+            return timestamp
+        except ValueError:
+            pass
+        return parser.parse(date_string).timestamp()
+
+    @classmethod
+    def conform_dict(cls, data: dict) -> None:
+        datetime_columns = [
+            "data_source_date_created",
+            "data_source_date_modified",
+            "data_source_date_processed",
+            "last_modified",
+        ]
+
+        json_dumps_fields = ["languages", "data_source_permissions_data"]
+
+        # TODO: milvus sdk doesn't seem to support defaults via the schema yet,
+        #  remove once that gets updated
+        defaults = {"is_continuation": False}
+
+        if metadata := data.pop("metadata", None):
+            data.update(flatten_dict(metadata, keys_to_omit=["data_source_record_locator"]))
+        for datetime_column in datetime_columns:
+            if datetime_column in data:
+                data[datetime_column] = cls.parse_date_string(data[datetime_column])
+        for json_dumps_field in json_dumps_fields:
+            if json_dumps_field in data:
+                data[json_dumps_field] = json.dumps(data[json_dumps_field])
+        for default in defaults:
+            if default not in data:
+                data[default] = defaults[default]
+
+    def run(
+        self,
+        elements_filepath: Path,
+        file_data: FileData,
+        output_dir: Path,
+        output_filename: str,
+        **kwargs: Any,
+    ) -> Path:
+        with open(elements_filepath) as elements_file:
+            elements_contents: list[dict[str, Any]] = json.load(elements_file)
+        for element in elements_contents:
+            self.conform_dict(data=element)
+
+        output_path = Path(output_dir) / Path(f"{output_filename}.json")
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        with output_path.open("w") as output_file:
+            json.dump(elements_contents, output_file, indent=2)
+        return output_path
+
+
+@dataclass
+class MilvusUploaderConfig(UploaderConfig):
+    collection_name: str
+    num_of_processes: int = 4
+
+
+@dataclass
+class MilvusUploader(Uploader):
+    connection_config: MilvusConnectionConfig
+    upload_config: MilvusUploaderConfig
+    connector_type: str = CONNECTOR_TYPE
+
+    def upload(self, content: UploadContent) -> None:
+        file_extension = content.path.suffix
+        if file_extension == ".json":
+            self.upload_json(content=content)
+        elif file_extension == ".csv":
+            self.upload_csv(content=content)
+        else:
+            raise ValueError(f"Unsupported file extension: {file_extension}")
+
+    @requires_dependencies(["pymilvus"], extras="milvus")
+    def insert_results(self, data: Union[dict, list[dict]]):
+        from pymilvus import MilvusException
+
+        logger.debug(
+            f"uploading {len(data)} entries to {self.connection_config.db_name} "
+            f"db in collection {self.upload_config.collection_name}"
+        )
+        client = self.connection_config.get_client()
+
+        try:
+            res = client.insert(collection_name=self.upload_config.collection_name, data=data)
+        except MilvusException as milvus_exception:
+            raise WriteError("failed to upload records to milvus") from milvus_exception
+        if "err_count" in res and isinstance(res["err_count"], int) and res["err_count"] > 0:
+            err_count = res["err_count"]
+            raise WriteError(f"failed to upload {err_count} docs")
+
+    def upload_csv(self, content: UploadContent) -> None:
+        df = pd.read_csv(content.path)
+        data = df.to_dict(orient="records")
+        self.insert_results(data=data)
+
+    def upload_json(self, content: UploadContent) -> None:
+        with content.path.open("r") as file:
+            data: list[dict] = json.load(file)
+        self.insert_results(data=data)
+
+    def run(self, contents: list[UploadContent], **kwargs: Any) -> None:
+        if self.upload_config.num_of_processes == 1:
+            for content in contents:
+                self.upload(content=content)
+
+        else:
+            with mp.Pool(
+                processes=self.upload_config.num_of_processes,
+            ) as pool:
+                pool.map(self.upload, contents)
+
+
+milvus_destination_entry = DestinationRegistryEntry(
+    connection_config=MilvusConnectionConfig,
+    uploader=MilvusUploader,
+    uploader_config=MilvusUploaderConfig,
+    upload_stager=MilvusUploadStager,
+    upload_stager_config=MilvusUploadStagerConfig,
+)

{unstructured-ingest-0.0.0 → unstructured-ingest-0.0.1}/unstructured_ingest/v2/processes/connectors/pinecone.py

@@ -5,10 +5,6 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Optional
 
-from unstructured.ingest.v2.logger import logger
-from unstructured.ingest.v2.processes.connector_registry import (
-    DestinationRegistryEntry,
-)
 from unstructured.staging.base import flatten_dict
 from unstructured.utils import requires_dependencies
 
@@ -24,6 +20,10 @@ from unstructured_ingest.v2.interfaces import (
     UploadStager,
     UploadStagerConfig,
 )
+from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.v2.processes.connector_registry import (
+    DestinationRegistryEntry,
+)
 
 if TYPE_CHECKING:
     from pinecone import Index as PineconeIndex

{unstructured-ingest-0.0.0 → unstructured-ingest-0.0.1}/unstructured_ingest.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unstructured-ingest
-Version: 0.0.0
+Version: 0.0.1
 Summary: A library that prepares raw documents for downstream ML tasks.
 Home-page: https://github.com/Unstructured-IO/unstructured-ingest
 Author: Unstructured Technologies
@@ -56,6 +56,7 @@ Provides-Extra: google-drive
 Provides-Extra: hubspot
 Provides-Extra: jira
 Provides-Extra: kafka
+Provides-Extra: milvus
 Provides-Extra: mongodb
 Provides-Extra: notion
 Provides-Extra: onedrive

{unstructured-ingest-0.0.0 → unstructured-ingest-0.0.1}/unstructured_ingest.egg-info/SOURCES.txt

@@ -286,6 +286,7 @@ unstructured_ingest/v2/cli/cmds/databricks_volumes.py
 unstructured_ingest/v2/cli/cmds/elasticsearch.py
 unstructured_ingest/v2/cli/cmds/google_drive.py
 unstructured_ingest/v2/cli/cmds/local.py
+unstructured_ingest/v2/cli/cmds/milvus.py
 unstructured_ingest/v2/cli/cmds/mongodb.py
 unstructured_ingest/v2/cli/cmds/onedrive.py
 unstructured_ingest/v2/cli/cmds/opensearch.py
@@ -344,6 +345,7 @@ unstructured_ingest/v2/processes/connectors/databricks_volumes.py
 unstructured_ingest/v2/processes/connectors/elasticsearch.py
 unstructured_ingest/v2/processes/connectors/google_drive.py
 unstructured_ingest/v2/processes/connectors/local.py
+unstructured_ingest/v2/processes/connectors/milvus.py
 unstructured_ingest/v2/processes/connectors/mongodb.py
 unstructured_ingest/v2/processes/connectors/onedrive.py
 unstructured_ingest/v2/processes/connectors/opensearch.py

{unstructured-ingest-0.0.0 → unstructured-ingest-0.0.1}/unstructured_ingest.egg-info/requires.txt

@@ -1,5 +1,5 @@
-pandas
 python-dateutil
+pandas
 unstructured
 
 [airtable]
@@ -16,8 +16,8 @@ fsspec
 azure-search-documents
 
 [bedrock]
-langchain-community
 boto3
+langchain-community
 
 [biomed]
 bs4
@@ -27,8 +27,8 @@ fsspec
 boxfs
 
 [chroma]
-typer<=0.9.0
 chromadb
+typer<=0.9.0
 importlib-metadata>=7.1.0
 
 [clarifai]
@@ -64,29 +64,29 @@ fsspec
 elasticsearch[async]
 
 [embed-huggingface]
-huggingface
-langchain-community
 sentence_transformers
+langchain-community
+huggingface
 
 [embed-octoai]
-tiktoken
 openai
+tiktoken
 
 [embed-vertexai]
-langchain-community
 langchain
+langchain-community
 langchain-google-vertexai
 
 [embed-voyageai]
-langchain-voyageai
 langchain
+langchain-voyageai
 
 [epub]
 unstructured[epub]
 
 [gcs]
-gcsfs
 fsspec
+gcsfs
 bs4
 
 [github]
@@ -111,6 +111,9 @@ confluent-kafka
 [md]
 unstructured[md]
 
+[milvus]
+pymilvus
+
 [mongodb]
 pymongo
 
@@ -125,14 +128,14 @@ htmlBuilder
 unstructured[odt]
 
 [onedrive]
+bs4
 msal
 Office365-REST-Python-Client
-bs4
 
 [openai]
+openai
 langchain-community
 tiktoken
-openai
 
 [opensearch]
 opensearch-py
@@ -172,8 +175,8 @@ unstructured[rst]
 unstructured[rtf]
 
 [s3]
-s3fs
 fsspec
+s3fs
 
 [salesforce]
 simple-salesforce

unstructured-ingest-0.0.0/unstructured_ingest/__version__.py

@@ -1 +0,0 @@
-__version__ = "0.0.0"  # pragma: no cover