unstructured-ingest 0.6.4__py3-none-any.whl → 0.7.0__py3-none-any.whl

unstructured_ingest/{v2/processes → processes}/partitioner.py

@@ -5,12 +5,12 @@ from typing import Any, Optional
 
 from pydantic import BaseModel, Field, SecretStr
 
+from unstructured_ingest.errors_v2 import UserError
+from unstructured_ingest.interfaces.process import BaseProcess
+from unstructured_ingest.logger import logger
+from unstructured_ingest.unstructured_api import call_api_async
 from unstructured_ingest.utils.data_prep import flatten_dict
 from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.errors import UserError
-from unstructured_ingest.v2.interfaces.process import BaseProcess
-from unstructured_ingest.v2.logger import logger
-from unstructured_ingest.v2.unstructured_api import call_api_async
 
 
 class PartitionerConfig(BaseModel):
@@ -34,7 +34,7 @@ class PartitionerConfig(BaseModel):
         default=None, description="Additional values to pass through to partition()"
     )
     skip_infer_table_types: Optional[list[str]] = Field(
-        default=None, description="Optional list of document types to skip table extraction on"
+        default=None, description="Optional list of document data_types to skip table extraction on"
     )
     fields_include: list[str] = Field(
         default_factory=lambda: ["element_id", "text", "type", "metadata", "embeddings"],
@@ -52,7 +52,7 @@ class PartitionerConfig(BaseModel):
     )
     element_exclude: list[str] = Field(
         default_factory=list,
-        description="If set, drop the specified element types, if they exist.",
+        description="If set, drop the specified element data_types, if they exist.",
     )
     metadata_include: list[str] = Field(
         default_factory=list,

unstructured_ingest/{v2/processes → processes}/uncompress.py

@@ -7,10 +7,10 @@ from uuid import NAMESPACE_DNS, uuid5
 
 from pydantic import BaseModel
 
+from unstructured_ingest.data_types.file_data import FileData, SourceIdentifiers
+from unstructured_ingest.interfaces.process import BaseProcess
+from unstructured_ingest.logger import logger
 from unstructured_ingest.utils.compression import TAR_FILE_EXT, ZIP_FILE_EXT, uncompress_file
-from unstructured_ingest.v2.interfaces.process import BaseProcess
-from unstructured_ingest.v2.logger import logger
-from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 
 class UncompressConfig(BaseModel):

unstructured_ingest/{v2/processes → processes}/utils/blob_storage.py

@@ -2,9 +2,9 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any
 
+from unstructured_ingest.data_types.file_data import FileData
+from unstructured_ingest.interfaces import UploadStager, UploadStagerConfig
 from unstructured_ingest.utils.data_prep import get_data, write_data
-from unstructured_ingest.v2.interfaces import UploadStager, UploadStagerConfig
-from unstructured_ingest.v2.types.file_data import FileData
 
 
 class BlobStoreUploadStagerConfig(UploadStagerConfig):

unstructured_ingest/{v2/unstructured_api.py → unstructured_api.py}

@@ -2,8 +2,8 @@ from dataclasses import fields
 from pathlib import Path
 from typing import TYPE_CHECKING, Optional
 
-from unstructured_ingest.v2.errors import ProviderError, QuotaError, UserAuthError, UserError
-from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.errors_v2 import ProviderError, QuotaError, UserAuthError, UserError
+from unstructured_ingest.logger import logger
 
 if TYPE_CHECKING:
     from unstructured_client.models.operations import PartitionRequest

unstructured_ingest/utils/compression.py

@@ -1,19 +1,10 @@
-import copy
 import os
 import sys
 import tarfile
 import zipfile
-from dataclasses import dataclass
 from pathlib import Path
-from typing import List, Optional
+from typing import Optional
 
-from unstructured_ingest.connector.local import LocalSourceConnector, SimpleLocalConfig
-from unstructured_ingest.interfaces import (
-    BaseConnectorConfig,
-    BaseSingleIngestDoc,
-    ProcessorConfig,
-    ReadConfig,
-)
 from unstructured_ingest.logger import logger
 
 ZIP_FILE_EXT = [".zip"]
@@ -78,41 +69,3 @@ def uncompress_tar_file(tar_filename: str, path: Optional[str] = None) -> str:
             )
         tfile.extractall(path=path)
     return path
-
-
-@dataclass
-class CompressionSourceConnectorMixin:
-    processor_config: ProcessorConfig
-    read_config: ReadConfig
-    connector_config: BaseConnectorConfig
-
-    def process_compressed_doc(self, doc: BaseSingleIngestDoc) -> List[BaseSingleIngestDoc]:
-        """
-        Utility function which helps process compressed files. Extracts the contents and returns
-        generated ingest docs via local source connector
-        """
-        # Download the raw file to local
-        doc.get_file()
-        path = uncompress_file(filename=str(doc.filename))
-        new_read_configs = copy.copy(self.read_config)
-        new_process_configs = copy.copy(self.processor_config)
-        relative_path = path.replace(self.read_config.download_dir, "")
-
-        if self.processor_config.output_dir.endswith(os.sep):
-            new_process_configs.output_dir = f"{self.processor_config.output_dir}{relative_path}"
-        else:
-            new_process_configs.output_dir = (
-                f"{self.processor_config.output_dir}{os.sep}{relative_path}"
-            )
-
-        local_connector = LocalSourceConnector(
-            connector_config=SimpleLocalConfig(
-                input_path=path,
-                recursive=True,
-            ),
-            read_config=new_read_configs,
-            processor_config=new_process_configs,
-        )
-        logger.info(f"created local source connector: {local_connector.to_json()}")
-        local_connector.initialize()
-        return local_connector.get_ingest_docs()

unstructured_ingest/utils/data_prep.py

@@ -3,10 +3,12 @@ import json
 from datetime import datetime
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Generator, Iterable, Optional, Sequence, TypeVar, Union, cast
+from uuid import NAMESPACE_DNS, uuid5
 
+from unstructured_ingest.data_types.file_data import FileData
+from unstructured_ingest.logger import logger
 from unstructured_ingest.utils import ndjson
 from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.logger import logger
 
 if TYPE_CHECKING:
     from pandas import DataFrame
@@ -230,3 +232,9 @@ def get_data_df(path: Path) -> "DataFrame":
             return df
         else:
             raise ValueError(f"Unsupported file type: {path}")
+
+
+def get_enhanced_element_id(element_dict: dict, file_data: FileData) -> str:
+    element_id = element_dict.get("element_id")
+    new_data = f"{element_id}{file_data.identifier}"
+    return str(uuid5(NAMESPACE_DNS, new_data))

unstructured_ingest/utils/html.py

@@ -6,10 +6,10 @@ from uuid import NAMESPACE_DNS, uuid5
 
 from pydantic import BaseModel, Field
 
+from unstructured_ingest.data_types.file_data import FileData, SourceIdentifiers
+from unstructured_ingest.interfaces import DownloadResponse
+from unstructured_ingest.logger import logger
 from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.interfaces import DownloadResponse
-from unstructured_ingest.v2.logger import logger
-from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 if TYPE_CHECKING:
     from bs4.element import Tag

unstructured_ingest/{v2/utils.py → utils/pydantic_models.py}

@@ -3,13 +3,10 @@ from datetime import datetime
 from inspect import isclass
 from pathlib import Path
 from typing import Any
-from uuid import NAMESPACE_DNS, uuid5
 
 from pydantic import BaseModel
 from pydantic.types import _SecretBase
 
-from unstructured_ingest.v2.types.file_data import FileData
-
 
 def is_secret(value: Any) -> bool:
     # Case Secret[int]
@@ -53,9 +50,3 @@ def serialize_base_model_json(model: BaseModel, **json_kwargs) -> str:
 
     # Support json dumps kwargs such as sort_keys
     return json.dumps(model_dict, default=json_serial, **json_kwargs)
-
-
-def get_enhanced_element_id(element_dict: dict, file_data: FileData) -> str:
-    element_id = element_dict.get("element_id")
-    new_data = f"{element_id}{file_data.identifier}"
-    return str(uuid5(NAMESPACE_DNS, new_data))

unstructured_ingest/utils/string_and_date_utils.py

@@ -5,7 +5,7 @@ from datetime import datetime
 
 from dateutil import parser
 
-from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.logger import logger
 
 
 def json_to_dict(json_string: str) -> t.Union[str, t.Dict[str, t.Any]]:

{unstructured_ingest-0.6.4.dist-info → unstructured_ingest-0.7.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: unstructured-ingest
-Version: 0.6.4
+Version: 0.7.0
 Summary: A library that prepares raw documents for downstream ML tasks.
 Home-page: https://github.com/Unstructured-IO/unstructured-ingest
 Author: Unstructured Technologies
@@ -22,12 +22,12 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.9.0,<3.14
 Description-Content-Type: text/markdown
 License-File: LICENSE.md
+Requires-Dist: click
+Requires-Dist: python-dateutil
 Requires-Dist: pydantic>=2.7
 Requires-Dist: opentelemetry-sdk
 Requires-Dist: tqdm
-Requires-Dist: click
 Requires-Dist: dataclasses_json
-Requires-Dist: python-dateutil
 Requires-Dist: numpy
 Requires-Dist: pandas
 Provides-Extra: remote
@@ -112,13 +112,13 @@ Requires-Dist: azure-search-documents; extra == "azure-ai-search"
 Requires-Dist: numpy; extra == "azure-ai-search"
 Requires-Dist: pandas; extra == "azure-ai-search"
 Provides-Extra: biomed
-Requires-Dist: requests; extra == "biomed"
 Requires-Dist: bs4; extra == "biomed"
+Requires-Dist: requests; extra == "biomed"
 Requires-Dist: numpy; extra == "biomed"
 Requires-Dist: pandas; extra == "biomed"
 Provides-Extra: box
-Requires-Dist: fsspec; extra == "box"
 Requires-Dist: boxfs; extra == "box"
+Requires-Dist: fsspec; extra == "box"
 Requires-Dist: numpy; extra == "box"
 Requires-Dist: pandas; extra == "box"
 Provides-Extra: chroma
@@ -130,8 +130,8 @@ Requires-Dist: clarifai; extra == "clarifai"
 Requires-Dist: numpy; extra == "clarifai"
 Requires-Dist: pandas; extra == "clarifai"
 Provides-Extra: confluence
-Requires-Dist: requests; extra == "confluence"
 Requires-Dist: atlassian-python-api; extra == "confluence"
+Requires-Dist: requests; extra == "confluence"
 Requires-Dist: numpy; extra == "confluence"
 Requires-Dist: pandas; extra == "confluence"
 Provides-Extra: couchbase
@@ -139,8 +139,8 @@ Requires-Dist: couchbase; extra == "couchbase"
 Requires-Dist: numpy; extra == "couchbase"
 Requires-Dist: pandas; extra == "couchbase"
 Provides-Extra: delta-table
-Requires-Dist: deltalake; extra == "delta-table"
 Requires-Dist: boto3; extra == "delta-table"
+Requires-Dist: deltalake; extra == "delta-table"
 Requires-Dist: numpy; extra == "delta-table"
 Requires-Dist: pandas; extra == "delta-table"
 Provides-Extra: discord
@@ -167,8 +167,8 @@ Requires-Dist: gcsfs; extra == "gcs"
 Requires-Dist: numpy; extra == "gcs"
 Requires-Dist: pandas; extra == "gcs"
 Provides-Extra: github
-Requires-Dist: requests; extra == "github"
 Requires-Dist: pygithub>1.58.0; extra == "github"
+Requires-Dist: requests; extra == "github"
 Requires-Dist: numpy; extra == "github"
 Requires-Dist: pandas; extra == "github"
 Provides-Extra: gitlab
@@ -185,10 +185,10 @@ Requires-Dist: urllib3; extra == "hubspot"
 Requires-Dist: numpy; extra == "hubspot"
 Requires-Dist: pandas; extra == "hubspot"
 Provides-Extra: ibm-watsonx-s3
+Requires-Dist: pyarrow; extra == "ibm-watsonx-s3"
+Requires-Dist: httpx; extra == "ibm-watsonx-s3"
 Requires-Dist: tenacity; extra == "ibm-watsonx-s3"
 Requires-Dist: pyiceberg; extra == "ibm-watsonx-s3"
-Requires-Dist: httpx; extra == "ibm-watsonx-s3"
-Requires-Dist: pyarrow; extra == "ibm-watsonx-s3"
 Requires-Dist: numpy; extra == "ibm-watsonx-s3"
 Requires-Dist: pandas; extra == "ibm-watsonx-s3"
 Provides-Extra: jira
@@ -216,22 +216,22 @@ Requires-Dist: pymongo; extra == "mongodb"
 Requires-Dist: numpy; extra == "mongodb"
 Requires-Dist: pandas; extra == "mongodb"
 Provides-Extra: neo4j
-Requires-Dist: neo4j-rust-ext; extra == "neo4j"
 Requires-Dist: networkx; extra == "neo4j"
 Requires-Dist: cymple; extra == "neo4j"
+Requires-Dist: neo4j-rust-ext; extra == "neo4j"
 Requires-Dist: numpy; extra == "neo4j"
 Requires-Dist: pandas; extra == "neo4j"
 Provides-Extra: notion
 Requires-Dist: htmlBuilder; extra == "notion"
-Requires-Dist: notion-client; extra == "notion"
 Requires-Dist: httpx; extra == "notion"
+Requires-Dist: notion-client; extra == "notion"
 Requires-Dist: backoff; extra == "notion"
 Requires-Dist: numpy; extra == "notion"
 Requires-Dist: pandas; extra == "notion"
 Provides-Extra: onedrive
-Requires-Dist: Office365-REST-Python-Client; extra == "onedrive"
 Requires-Dist: bs4; extra == "onedrive"
 Requires-Dist: msal; extra == "onedrive"
+Requires-Dist: Office365-REST-Python-Client; extra == "onedrive"
 Requires-Dist: numpy; extra == "onedrive"
 Requires-Dist: pandas; extra == "onedrive"
 Provides-Extra: opensearch
@@ -239,8 +239,8 @@ Requires-Dist: opensearch-py; extra == "opensearch"
 Requires-Dist: numpy; extra == "opensearch"
 Requires-Dist: pandas; extra == "opensearch"
 Provides-Extra: outlook
-Requires-Dist: Office365-REST-Python-Client; extra == "outlook"
 Requires-Dist: msal; extra == "outlook"
+Requires-Dist: Office365-REST-Python-Client; extra == "outlook"
 Requires-Dist: numpy; extra == "outlook"
 Requires-Dist: pandas; extra == "outlook"
 Provides-Extra: pinecone
@@ -269,8 +269,8 @@ Requires-Dist: s3fs; extra == "s3"
 Requires-Dist: numpy; extra == "s3"
 Requires-Dist: pandas; extra == "s3"
 Provides-Extra: sharepoint
-Requires-Dist: Office365-REST-Python-Client; extra == "sharepoint"
 Requires-Dist: msal; extra == "sharepoint"
+Requires-Dist: Office365-REST-Python-Client; extra == "sharepoint"
 Requires-Dist: numpy; extra == "sharepoint"
 Requires-Dist: pandas; extra == "sharepoint"
 Provides-Extra: salesforce
@@ -287,8 +287,8 @@ Requires-Dist: slack_sdk[optional]; extra == "slack"
 Requires-Dist: numpy; extra == "slack"
 Requires-Dist: pandas; extra == "slack"
 Provides-Extra: snowflake
-Requires-Dist: psycopg2-binary; extra == "snowflake"
 Requires-Dist: snowflake-connector-python; extra == "snowflake"
+Requires-Dist: psycopg2-binary; extra == "snowflake"
 Requires-Dist: numpy; extra == "snowflake"
 Requires-Dist: pandas; extra == "snowflake"
 Provides-Extra: wikipedia
@@ -312,15 +312,15 @@ Requires-Dist: singlestoredb; extra == "singlestore"
 Requires-Dist: numpy; extra == "singlestore"
 Requires-Dist: pandas; extra == "singlestore"
 Provides-Extra: vectara
-Requires-Dist: requests; extra == "vectara"
 Requires-Dist: httpx; extra == "vectara"
+Requires-Dist: requests; extra == "vectara"
 Requires-Dist: aiofiles; extra == "vectara"
 Requires-Dist: numpy; extra == "vectara"
 Requires-Dist: pandas; extra == "vectara"
 Provides-Extra: vastdb
-Requires-Dist: vastdb; extra == "vastdb"
 Requires-Dist: ibis; extra == "vastdb"
 Requires-Dist: pyarrow; extra == "vastdb"
+Requires-Dist: vastdb; extra == "vastdb"
 Requires-Dist: numpy; extra == "vastdb"
 Requires-Dist: pandas; extra == "vastdb"
 Provides-Extra: zendesk
@@ -334,8 +334,8 @@ Requires-Dist: sentence-transformers; extra == "embed-huggingface"
 Requires-Dist: numpy; extra == "embed-huggingface"
 Requires-Dist: pandas; extra == "embed-huggingface"
 Provides-Extra: embed-octoai
-Requires-Dist: openai; extra == "embed-octoai"
 Requires-Dist: tiktoken; extra == "embed-octoai"
+Requires-Dist: openai; extra == "embed-octoai"
 Requires-Dist: numpy; extra == "embed-octoai"
 Requires-Dist: pandas; extra == "embed-octoai"
 Provides-Extra: embed-vertexai
@@ -351,13 +351,13 @@ Requires-Dist: mixedbread-ai; extra == "embed-mixedbreadai"
 Requires-Dist: numpy; extra == "embed-mixedbreadai"
 Requires-Dist: pandas; extra == "embed-mixedbreadai"
 Provides-Extra: openai
-Requires-Dist: openai; extra == "openai"
 Requires-Dist: tiktoken; extra == "openai"
+Requires-Dist: openai; extra == "openai"
 Requires-Dist: numpy; extra == "openai"
 Requires-Dist: pandas; extra == "openai"
 Provides-Extra: bedrock
-Requires-Dist: aioboto3; extra == "bedrock"
 Requires-Dist: boto3; extra == "bedrock"
+Requires-Dist: aioboto3; extra == "bedrock"
 Requires-Dist: numpy; extra == "bedrock"
 Requires-Dist: pandas; extra == "bedrock"
 Provides-Extra: togetherai