unstructured-ingest 0.5.25__py3-none-any.whl → 0.6.1__py3-none-any.whl

unstructured_ingest/v2/processes/connectors/milvus.py

@@ -13,7 +13,6 @@ from unstructured_ingest.v2.constants import RECORD_ID_LABEL
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
     ConnectionConfig,
-    FileData,
     Uploader,
     UploaderConfig,
     UploadStager,
@@ -23,6 +22,7 @@ from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connector_registry import (
     DestinationRegistryEntry,
 )
+from unstructured_ingest.v2.types.file_data import FileData
 
 if TYPE_CHECKING:
     from pymilvus import MilvusClient

unstructured_ingest/v2/processes/connectors/mongodb.py

@@ -13,17 +13,12 @@ from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.constants import RECORD_ID_LABEL
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
-    BatchFileData,
-    BatchItem,
     ConnectionConfig,
     Downloader,
     DownloaderConfig,
     DownloadResponse,
-    FileData,
-    FileDataSourceMetadata,
     Indexer,
     IndexerConfig,
-    SourceIdentifiers,
     Uploader,
     UploaderConfig,
     download_responses,
@@ -33,6 +28,13 @@ from unstructured_ingest.v2.processes.connector_registry import (
     DestinationRegistryEntry,
     SourceRegistryEntry,
 )
+from unstructured_ingest.v2.types.file_data import (
+    BatchFileData,
+    BatchItem,
+    FileData,
+    FileDataSourceMetadata,
+    SourceIdentifiers,
+)
 
 if TYPE_CHECKING:
     from pymongo import MongoClient

unstructured_ingest/v2/processes/connectors/neo4j.py

@@ -18,7 +18,6 @@ from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
     ConnectionConfig,
-    FileData,
     Uploader,
     UploaderConfig,
     UploadStager,
@@ -28,6 +27,7 @@ from unstructured_ingest.v2.processes.connector_registry import (
     DestinationRegistryEntry,
 )
 from unstructured_ingest.v2.processes.connectors.utils import format_and_truncate_orig_elements
+from unstructured_ingest.v2.types.file_data import FileData
 
 SimilarityFunction = Literal["cosine"]
 

unstructured_ingest/v2/processes/connectors/notion/connector.py

@@ -12,14 +12,16 @@ from unstructured_ingest.v2.interfaces import (
     Downloader,
     DownloaderConfig,
     DownloadResponse,
-    FileData,
-    FileDataSourceMetadata,
     Indexer,
     IndexerConfig,
-    SourceIdentifiers,
 )
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connector_registry import SourceRegistryEntry
+from unstructured_ingest.v2.types.file_data import (
+    FileData,
+    FileDataSourceMetadata,
+    SourceIdentifiers,
+)
 
 if TYPE_CHECKING:
     from unstructured_ingest.v2.processes.connectors.notion.client import Client

unstructured_ingest/v2/processes/connectors/onedrive.py

@@ -22,11 +22,8 @@ from unstructured_ingest.v2.interfaces import (
     Downloader,
     DownloaderConfig,
     DownloadResponse,
-    FileData,
-    FileDataSourceMetadata,
     Indexer,
     IndexerConfig,
-    SourceIdentifiers,
     Uploader,
     UploaderConfig,
 )
@@ -39,6 +36,11 @@ from unstructured_ingest.v2.processes.utils.blob_storage import (
     BlobStoreUploadStager,
     BlobStoreUploadStagerConfig,
 )
+from unstructured_ingest.v2.types.file_data import (
+    FileData,
+    FileDataSourceMetadata,
+    SourceIdentifiers,
+)
 
 if TYPE_CHECKING:
     from office365.graph_client import GraphClient

unstructured_ingest/v2/processes/connectors/outlook.py

@@ -16,12 +16,15 @@ from unstructured_ingest.v2.interfaces import (
     Downloader,
     DownloaderConfig,
     DownloadResponse,
-    FileData,
     Indexer,
     IndexerConfig,
 )
-from unstructured_ingest.v2.interfaces.file_data import FileDataSourceMetadata, SourceIdentifiers
 from unstructured_ingest.v2.processes.connector_registry import SourceRegistryEntry
+from unstructured_ingest.v2.types.file_data import (
+    FileData,
+    FileDataSourceMetadata,
+    SourceIdentifiers,
+)
 
 MAX_EMAILS_PER_FOLDER = 1_000_000  # Maximum number of emails per folder
 

unstructured_ingest/v2/processes/connectors/pinecone.py

@@ -13,7 +13,6 @@ from unstructured_ingest.v2.errors import UserError
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
     ConnectionConfig,
-    FileData,
     UploaderConfig,
     UploadStager,
     UploadStagerConfig,
@@ -21,6 +20,7 @@ from unstructured_ingest.v2.interfaces import (
 )
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connector_registry import DestinationRegistryEntry
+from unstructured_ingest.v2.types.file_data import FileData
 from unstructured_ingest.v2.utils import get_enhanced_element_id
 
 if TYPE_CHECKING:

unstructured_ingest/v2/processes/connectors/qdrant/qdrant.py

@@ -13,13 +13,13 @@ from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
     ConnectionConfig,
-    FileData,
     Uploader,
     UploaderConfig,
     UploadStager,
     UploadStagerConfig,
 )
 from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.v2.types.file_data import FileData
 from unstructured_ingest.v2.utils import get_enhanced_element_id
 
 if TYPE_CHECKING:

unstructured_ingest/v2/processes/connectors/redisdb.py

@@ -11,12 +11,12 @@ from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
     ConnectionConfig,
-    FileData,
     Uploader,
     UploaderConfig,
 )
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connector_registry import DestinationRegistryEntry
+from unstructured_ingest.v2.types.file_data import FileData
 
 if TYPE_CHECKING:
     from redis.asyncio import Redis

unstructured_ingest/v2/processes/connectors/salesforce.py

@@ -28,16 +28,18 @@ from unstructured_ingest.v2.interfaces import (
     Downloader,
     DownloaderConfig,
     DownloadResponse,
-    FileData,
-    FileDataSourceMetadata,
     Indexer,
     IndexerConfig,
-    SourceIdentifiers,
 )
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connector_registry import (
     SourceRegistryEntry,
 )
+from unstructured_ingest.v2.types.file_data import (
+    FileData,
+    FileDataSourceMetadata,
+    SourceIdentifiers,
+)
 
 
 class MissingCategoryError(Exception):

unstructured_ingest/v2/processes/connectors/sharepoint.py

@@ -11,9 +11,6 @@ from unstructured_ingest.error import (
     SourceConnectionNetworkError,
 )
 from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.interfaces import (
-    FileData,
-)
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connector_registry import (
     SourceRegistryEntry,
@@ -26,6 +23,9 @@ from unstructured_ingest.v2.processes.connectors.onedrive import (
     OnedriveIndexer,
     OnedriveIndexerConfig,
 )
+from unstructured_ingest.v2.types.file_data import (
+    FileData,
+)
 
 if TYPE_CHECKING:
     from office365.onedrive.driveitems.driveItem import DriveItem

unstructured_ingest/v2/processes/connectors/slack.py

@@ -20,12 +20,12 @@ from unstructured_ingest.v2.interfaces import (
     Indexer,
     IndexerConfig,
 )
-from unstructured_ingest.v2.interfaces.file_data import (
+from unstructured_ingest.v2.processes.connector_registry import SourceRegistryEntry
+from unstructured_ingest.v2.types.file_data import (
     FileData,
     FileDataSourceMetadata,
     SourceIdentifiers,
 )
-from unstructured_ingest.v2.processes.connector_registry import SourceRegistryEntry
 
 if TYPE_CHECKING:
     from slack_sdk import WebClient

unstructured_ingest/v2/processes/connectors/sql/databricks_delta_tables.py

@@ -7,7 +7,6 @@ from pydantic import Field, Secret
 
 from unstructured_ingest.utils.data_prep import split_dataframe
 from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.interfaces import FileData
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connector_registry import (
     DestinationRegistryEntry,
@@ -20,6 +19,7 @@ from unstructured_ingest.v2.processes.connectors.sql.sql import (
     SQLUploadStager,
     SQLUploadStagerConfig,
 )
+from unstructured_ingest.v2.types.file_data import FileData
 
 if TYPE_CHECKING:
     from databricks.sdk.core import oauth_service_principal

unstructured_ingest/v2/processes/connectors/sql/snowflake.py

@@ -7,7 +7,6 @@ from pydantic import Field, Secret
 
 from unstructured_ingest.utils.data_prep import split_dataframe
 from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.interfaces.file_data import FileData
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connector_registry import (
     DestinationRegistryEntry,
@@ -28,6 +27,7 @@ from unstructured_ingest.v2.processes.connectors.sql.sql import (
     SQLUploadStagerConfig,
     parse_date_string,
 )
+from unstructured_ingest.v2.types.file_data import FileData
 
 if TYPE_CHECKING:
     from pandas import DataFrame

unstructured_ingest/v2/processes/connectors/sql/sql.py

@@ -16,17 +16,12 @@ from unstructured_ingest.utils.data_prep import get_data, get_data_df, split_dat
 from unstructured_ingest.v2.constants import RECORD_ID_LABEL
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
-    BatchFileData,
-    BatchItem,
     ConnectionConfig,
     Downloader,
     DownloaderConfig,
     DownloadResponse,
-    FileData,
-    FileDataSourceMetadata,
     Indexer,
     IndexerConfig,
-    SourceIdentifiers,
     Uploader,
     UploaderConfig,
     UploadStager,
@@ -34,6 +29,13 @@ from unstructured_ingest.v2.interfaces import (
     download_responses,
 )
 from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.v2.types.file_data import (
+    BatchFileData,
+    BatchItem,
+    FileData,
+    FileDataSourceMetadata,
+    SourceIdentifiers,
+)
 from unstructured_ingest.v2.utils import get_enhanced_element_id
 
 if TYPE_CHECKING:

unstructured_ingest/v2/processes/connectors/sql/vastdb.py

@@ -8,9 +8,6 @@ from unstructured_ingest.error import DestinationConnectionError
 from unstructured_ingest.utils.data_prep import split_dataframe
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.constants import RECORD_ID_LABEL
-from unstructured_ingest.v2.interfaces import (
-    FileData,
-)
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connector_registry import (
     DestinationRegistryEntry,
@@ -29,6 +26,9 @@ from unstructured_ingest.v2.processes.connectors.sql.sql import (
     SQLUploadStager,
     SQLUploadStagerConfig,
 )
+from unstructured_ingest.v2.types.file_data import (
+    FileData,
+)
 from unstructured_ingest.v2.utils import get_enhanced_element_id
 
 if TYPE_CHECKING:

unstructured_ingest/v2/processes/connectors/vectara.py

@@ -14,7 +14,6 @@ from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
     ConnectionConfig,
-    FileData,
     Uploader,
     UploaderConfig,
     UploadStager,
@@ -22,6 +21,7 @@ from unstructured_ingest.v2.interfaces import (
 )
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connector_registry import DestinationRegistryEntry
+from unstructured_ingest.v2.types.file_data import FileData
 
 BASE_URL = "https://api.vectara.io/v2"
 

unstructured_ingest/v2/processes/connectors/weaviate/weaviate.py

@@ -16,13 +16,13 @@ from unstructured_ingest.v2.constants import RECORD_ID_LABEL
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
     ConnectionConfig,
-    FileData,
     UploaderConfig,
     UploadStager,
     UploadStagerConfig,
     VectorDBUploader,
 )
 from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.v2.types.file_data import FileData
 
 if TYPE_CHECKING:
     from weaviate.classes.init import Timeout

unstructured_ingest/v2/processes/connectors/zendesk/zendesk.py

@@ -16,14 +16,16 @@ from unstructured_ingest.v2.interfaces import (
     Downloader,
     DownloaderConfig,
     DownloadResponse,
-    FileData,
-    FileDataSourceMetadata,
     Indexer,
     IndexerConfig,
-    SourceIdentifiers,
 )
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connector_registry import SourceRegistryEntry
+from unstructured_ingest.v2.types.file_data import (
+    FileData,
+    FileDataSourceMetadata,
+    SourceIdentifiers,
+)
 
 from .client import ZendeskArticle, ZendeskClient, ZendeskTicket
 

unstructured_ingest/v2/processes/filter.py

@@ -5,9 +5,9 @@ from typing import Any, Callable, Optional
 
 from pydantic import BaseModel, Field
 
-from unstructured_ingest.v2.interfaces import FileData
 from unstructured_ingest.v2.interfaces.process import BaseProcess
 from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.v2.types.file_data import FileData
 
 
 class FiltererConfig(BaseModel):

unstructured_ingest/v2/processes/uncompress.py

@@ -8,9 +8,9 @@ from uuid import NAMESPACE_DNS, uuid5
 from pydantic import BaseModel
 
 from unstructured_ingest.utils.compression import TAR_FILE_EXT, ZIP_FILE_EXT, uncompress_file
-from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
 from unstructured_ingest.v2.interfaces.process import BaseProcess
 from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 
 class UncompressConfig(BaseModel):

unstructured_ingest/v2/processes/utils/blob_storage.py

@@ -3,7 +3,8 @@ from pathlib import Path
 from typing import Any
 
 from unstructured_ingest.utils.data_prep import get_data, write_data
-from unstructured_ingest.v2.interfaces import FileData, UploadStager, UploadStagerConfig
+from unstructured_ingest.v2.interfaces import UploadStager, UploadStagerConfig
+from unstructured_ingest.v2.types.file_data import FileData
 
 
 class BlobStoreUploadStagerConfig(UploadStagerConfig):

unstructured_ingest/v2/utils.py

@@ -8,7 +8,7 @@ from uuid import NAMESPACE_DNS, uuid5
 from pydantic import BaseModel
 from pydantic.types import _SecretBase
 
-from unstructured_ingest.v2.interfaces import FileData
+from unstructured_ingest.v2.types.file_data import FileData
 
 
 def is_secret(value: Any) -> bool:

{unstructured_ingest-0.5.25.dist-info → unstructured_ingest-0.6.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: unstructured-ingest
-Version: 0.5.25
+Version: 0.6.1
 Summary: A library that prepares raw documents for downstream ML tasks.
 Home-page: https://github.com/Unstructured-IO/unstructured-ingest
 Author: Unstructured Technologies
@@ -22,11 +22,11 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.9.0,<3.14
 Description-Content-Type: text/markdown
 License-File: LICENSE.md
+Requires-Dist: click
+Requires-Dist: tqdm
 Requires-Dist: opentelemetry-sdk
 Requires-Dist: python-dateutil
-Requires-Dist: click
 Requires-Dist: dataclasses_json
-Requires-Dist: tqdm
 Requires-Dist: pydantic>=2.7
 Requires-Dist: numpy
 Requires-Dist: pandas
@@ -117,8 +117,8 @@ Requires-Dist: bs4; extra == "biomed"
 Requires-Dist: numpy; extra == "biomed"
 Requires-Dist: pandas; extra == "biomed"
 Provides-Extra: box
-Requires-Dist: boxfs; extra == "box"
 Requires-Dist: fsspec; extra == "box"
+Requires-Dist: boxfs; extra == "box"
 Requires-Dist: numpy; extra == "box"
 Requires-Dist: pandas; extra == "box"
 Provides-Extra: chroma
@@ -130,8 +130,8 @@ Requires-Dist: clarifai; extra == "clarifai"
 Requires-Dist: numpy; extra == "clarifai"
 Requires-Dist: pandas; extra == "clarifai"
 Provides-Extra: confluence
-Requires-Dist: atlassian-python-api; extra == "confluence"
 Requires-Dist: requests; extra == "confluence"
+Requires-Dist: atlassian-python-api; extra == "confluence"
 Requires-Dist: numpy; extra == "confluence"
 Requires-Dist: pandas; extra == "confluence"
 Provides-Extra: couchbase
@@ -185,10 +185,10 @@ Requires-Dist: urllib3; extra == "hubspot"
 Requires-Dist: numpy; extra == "hubspot"
 Requires-Dist: pandas; extra == "hubspot"
 Provides-Extra: ibm-watsonx-s3
-Requires-Dist: tenacity; extra == "ibm-watsonx-s3"
+Requires-Dist: httpx; extra == "ibm-watsonx-s3"
 Requires-Dist: pyiceberg; extra == "ibm-watsonx-s3"
+Requires-Dist: tenacity; extra == "ibm-watsonx-s3"
 Requires-Dist: pyarrow; extra == "ibm-watsonx-s3"
-Requires-Dist: httpx; extra == "ibm-watsonx-s3"
 Requires-Dist: numpy; extra == "ibm-watsonx-s3"
 Requires-Dist: pandas; extra == "ibm-watsonx-s3"
 Provides-Extra: jira
@@ -222,16 +222,16 @@ Requires-Dist: neo4j-rust-ext; extra == "neo4j"
 Requires-Dist: numpy; extra == "neo4j"
 Requires-Dist: pandas; extra == "neo4j"
 Provides-Extra: notion
+Requires-Dist: httpx; extra == "notion"
 Requires-Dist: notion-client; extra == "notion"
-Requires-Dist: backoff; extra == "notion"
 Requires-Dist: htmlBuilder; extra == "notion"
-Requires-Dist: httpx; extra == "notion"
+Requires-Dist: backoff; extra == "notion"
 Requires-Dist: numpy; extra == "notion"
 Requires-Dist: pandas; extra == "notion"
 Provides-Extra: onedrive
-Requires-Dist: msal; extra == "onedrive"
 Requires-Dist: Office365-REST-Python-Client; extra == "onedrive"
 Requires-Dist: bs4; extra == "onedrive"
+Requires-Dist: msal; extra == "onedrive"
 Requires-Dist: numpy; extra == "onedrive"
 Requires-Dist: pandas; extra == "onedrive"
 Provides-Extra: opensearch
@@ -239,8 +239,8 @@ Requires-Dist: opensearch-py; extra == "opensearch"
 Requires-Dist: numpy; extra == "opensearch"
 Requires-Dist: pandas; extra == "opensearch"
 Provides-Extra: outlook
-Requires-Dist: msal; extra == "outlook"
 Requires-Dist: Office365-REST-Python-Client; extra == "outlook"
+Requires-Dist: msal; extra == "outlook"
 Requires-Dist: numpy; extra == "outlook"
 Requires-Dist: pandas; extra == "outlook"
 Provides-Extra: pinecone
@@ -264,13 +264,13 @@ Requires-Dist: redis; extra == "redis"
 Requires-Dist: numpy; extra == "redis"
 Requires-Dist: pandas; extra == "redis"
 Provides-Extra: s3
-Requires-Dist: fsspec; extra == "s3"
 Requires-Dist: s3fs; extra == "s3"
+Requires-Dist: fsspec; extra == "s3"
 Requires-Dist: numpy; extra == "s3"
 Requires-Dist: pandas; extra == "s3"
 Provides-Extra: sharepoint
-Requires-Dist: msal; extra == "sharepoint"
 Requires-Dist: Office365-REST-Python-Client; extra == "sharepoint"
+Requires-Dist: msal; extra == "sharepoint"
 Requires-Dist: numpy; extra == "sharepoint"
 Requires-Dist: pandas; extra == "sharepoint"
 Provides-Extra: salesforce
@@ -318,14 +318,14 @@ Requires-Dist: httpx; extra == "vectara"
 Requires-Dist: numpy; extra == "vectara"
 Requires-Dist: pandas; extra == "vectara"
 Provides-Extra: vastdb
-Requires-Dist: pyarrow; extra == "vastdb"
-Requires-Dist: ibis; extra == "vastdb"
 Requires-Dist: vastdb; extra == "vastdb"
+Requires-Dist: ibis; extra == "vastdb"
+Requires-Dist: pyarrow; extra == "vastdb"
 Requires-Dist: numpy; extra == "vastdb"
 Requires-Dist: pandas; extra == "vastdb"
 Provides-Extra: zendesk
-Requires-Dist: aiofiles; extra == "zendesk"
 Requires-Dist: httpx; extra == "zendesk"
+Requires-Dist: aiofiles; extra == "zendesk"
 Requires-Dist: bs4; extra == "zendesk"
 Requires-Dist: numpy; extra == "zendesk"
 Requires-Dist: pandas; extra == "zendesk"