unstructured-ingest 0.6.2__py3-none-any.whl → 0.7.0__py3-none-any.whl

unstructured_ingest/{v2/processes → processes}/connectors/weaviate/cloud.py

@@ -4,9 +4,8 @@ from typing import TYPE_CHECKING, Any, Generator, Optional
 
 from pydantic import Field, Secret
 
-from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.processes.connector_registry import DestinationRegistryEntry
-from unstructured_ingest.v2.processes.connectors.weaviate.weaviate import (
+from unstructured_ingest.processes.connector_registry import DestinationRegistryEntry
+from unstructured_ingest.processes.connectors.weaviate.weaviate import (
     WeaviateAccessConfig,
     WeaviateConnectionConfig,
     WeaviateUploader,
@@ -14,6 +13,7 @@ from unstructured_ingest.v2.processes.connectors.weaviate.weaviate import (
     WeaviateUploadStager,
     WeaviateUploadStagerConfig,
 )
+from unstructured_ingest.utils.dep_check import requires_dependencies
 
 if TYPE_CHECKING:
     from weaviate.auth import AuthCredentials

unstructured_ingest/{v2/processes → processes}/connectors/weaviate/embedded.py

@@ -4,9 +4,8 @@ from typing import TYPE_CHECKING, Generator, Optional
 
 from pydantic import Field, Secret
 
-from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.processes.connector_registry import DestinationRegistryEntry
-from unstructured_ingest.v2.processes.connectors.weaviate.weaviate import (
+from unstructured_ingest.processes.connector_registry import DestinationRegistryEntry
+from unstructured_ingest.processes.connectors.weaviate.weaviate import (
     WeaviateAccessConfig,
     WeaviateConnectionConfig,
     WeaviateUploader,
@@ -14,6 +13,7 @@ from unstructured_ingest.v2.processes.connectors.weaviate.weaviate import (
     WeaviateUploadStager,
     WeaviateUploadStagerConfig,
 )
+from unstructured_ingest.utils.dep_check import requires_dependencies
 
 if TYPE_CHECKING:
     from weaviate.client import WeaviateClient

unstructured_ingest/{v2/processes → processes}/connectors/weaviate/local.py

@@ -4,9 +4,8 @@ from typing import TYPE_CHECKING, Generator
 
 from pydantic import Field, Secret
 
-from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.processes.connector_registry import DestinationRegistryEntry
-from unstructured_ingest.v2.processes.connectors.weaviate.weaviate import (
+from unstructured_ingest.processes.connector_registry import DestinationRegistryEntry
+from unstructured_ingest.processes.connectors.weaviate.weaviate import (
     WeaviateAccessConfig,
     WeaviateConnectionConfig,
     WeaviateUploader,
@@ -14,6 +13,7 @@ from unstructured_ingest.v2.processes.connectors.weaviate.weaviate import (
     WeaviateUploadStager,
     WeaviateUploadStagerConfig,
 )
+from unstructured_ingest.utils.dep_check import requires_dependencies
 
 if TYPE_CHECKING:
     from weaviate.client import WeaviateClient

unstructured_ingest/{v2/processes → processes}/connectors/weaviate/weaviate.py

@@ -10,10 +10,9 @@ from typing import TYPE_CHECKING, Any, Generator, Optional
 from dateutil import parser
 from pydantic import Field, Secret
 
+from unstructured_ingest.data_types.file_data import FileData
 from unstructured_ingest.error import DestinationConnectionError, WriteError
-from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.constants import RECORD_ID_LABEL
-from unstructured_ingest.v2.interfaces import (
+from unstructured_ingest.interfaces import (
     AccessConfig,
     ConnectionConfig,
     UploaderConfig,
@@ -21,8 +20,9 @@ from unstructured_ingest.v2.interfaces import (
     UploadStagerConfig,
     VectorDBUploader,
 )
-from unstructured_ingest.v2.logger import logger
-from unstructured_ingest.v2.types.file_data import FileData
+from unstructured_ingest.logger import logger
+from unstructured_ingest.utils.constants import RECORD_ID_LABEL
+from unstructured_ingest.utils.dep_check import requires_dependencies
 
 if TYPE_CHECKING:
     from weaviate.classes.init import Timeout

unstructured_ingest/{v2/processes → processes}/connectors/zendesk/client.py

@@ -4,10 +4,10 @@ from typing import TYPE_CHECKING, Any, AsyncGenerator, Literal, Optional, Union
 
 from pydantic import BaseModel, Field, HttpUrl
 
+from unstructured_ingest.errors_v2 import ProviderError, RateLimitError, UserAuthError, UserError
+from unstructured_ingest.logger import logger
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.utils.string_and_date_utils import fix_unescaped_unicode
-from unstructured_ingest.v2.errors import ProviderError, RateLimitError, UserAuthError, UserError
-from unstructured_ingest.v2.logger import logger
 
 if TYPE_CHECKING:
     from httpx import AsyncClient, Client

unstructured_ingest/{v2/processes → processes}/connectors/zendesk/zendesk.py

@@ -8,9 +8,12 @@ from typing import Any, AsyncGenerator, Literal, Union
 
 from pydantic import BaseModel, Field, Secret
 
-from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.utils.html import HtmlMixin
-from unstructured_ingest.v2.interfaces import (
+from unstructured_ingest.data_types.file_data import (
+    FileData,
+    FileDataSourceMetadata,
+    SourceIdentifiers,
+)
+from unstructured_ingest.interfaces import (
     AccessConfig,
     ConnectionConfig,
     Downloader,
@@ -19,13 +22,10 @@ from unstructured_ingest.v2.interfaces import (
     Indexer,
     IndexerConfig,
 )
-from unstructured_ingest.v2.logger import logger
-from unstructured_ingest.v2.processes.connector_registry import SourceRegistryEntry
-from unstructured_ingest.v2.types.file_data import (
-    FileData,
-    FileDataSourceMetadata,
-    SourceIdentifiers,
-)
+from unstructured_ingest.logger import logger
+from unstructured_ingest.processes.connector_registry import SourceRegistryEntry
+from unstructured_ingest.utils.dep_check import requires_dependencies
+from unstructured_ingest.utils.html import HtmlMixin
 
 from .client import ZendeskArticle, ZendeskClient, ZendeskTicket
 

unstructured_ingest/{v2/processes → processes}/embedder.py

@@ -5,8 +5,8 @@ from typing import TYPE_CHECKING, Any, Literal, Optional
 
 from pydantic import BaseModel, Field, SecretStr
 
+from unstructured_ingest.interfaces.process import BaseProcess
 from unstructured_ingest.utils.data_prep import get_data
-from unstructured_ingest.v2.interfaces.process import BaseProcess
 
 if TYPE_CHECKING:
     from unstructured_ingest.embed.interfaces import BaseEmbeddingEncoder

unstructured_ingest/{v2/processes → processes}/filter.py

@@ -5,15 +5,15 @@ from typing import Any, Callable, Optional
 
 from pydantic import BaseModel, Field
 
-from unstructured_ingest.v2.interfaces.process import BaseProcess
-from unstructured_ingest.v2.logger import logger
-from unstructured_ingest.v2.types.file_data import FileData
+from unstructured_ingest.data_types.file_data import FileData
+from unstructured_ingest.interfaces.process import BaseProcess
+from unstructured_ingest.logger import logger
 
 
 class FiltererConfig(BaseModel):
     file_glob: Optional[list[str]] = Field(
         default=None,
-        description="file globs to limit which types of " "files are accepted",
+        description="file globs to limit which data_types of " "files are accepted",
         examples=["*.pdf", "*.html"],
     )
     max_file_size: Optional[int] = Field(

unstructured_ingest/{v2/processes → processes}/partitioner.py

@@ -5,12 +5,12 @@ from typing import Any, Optional
 
 from pydantic import BaseModel, Field, SecretStr
 
+from unstructured_ingest.errors_v2 import UserError
+from unstructured_ingest.interfaces.process import BaseProcess
+from unstructured_ingest.logger import logger
+from unstructured_ingest.unstructured_api import call_api_async
 from unstructured_ingest.utils.data_prep import flatten_dict
 from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.errors import UserError
-from unstructured_ingest.v2.interfaces.process import BaseProcess
-from unstructured_ingest.v2.logger import logger
-from unstructured_ingest.v2.unstructured_api import call_api_async
 
 
 class PartitionerConfig(BaseModel):
@@ -34,7 +34,7 @@ class PartitionerConfig(BaseModel):
         default=None, description="Additional values to pass through to partition()"
     )
     skip_infer_table_types: Optional[list[str]] = Field(
-        default=None, description="Optional list of document types to skip table extraction on"
+        default=None, description="Optional list of document data_types to skip table extraction on"
     )
     fields_include: list[str] = Field(
         default_factory=lambda: ["element_id", "text", "type", "metadata", "embeddings"],
@@ -52,7 +52,7 @@ class PartitionerConfig(BaseModel):
     )
     element_exclude: list[str] = Field(
         default_factory=list,
-        description="If set, drop the specified element types, if they exist.",
+        description="If set, drop the specified element data_types, if they exist.",
     )
     metadata_include: list[str] = Field(
         default_factory=list,

unstructured_ingest/{v2/processes → processes}/uncompress.py

@@ -7,10 +7,10 @@ from uuid import NAMESPACE_DNS, uuid5
 
 from pydantic import BaseModel
 
+from unstructured_ingest.data_types.file_data import FileData, SourceIdentifiers
+from unstructured_ingest.interfaces.process import BaseProcess
+from unstructured_ingest.logger import logger
 from unstructured_ingest.utils.compression import TAR_FILE_EXT, ZIP_FILE_EXT, uncompress_file
-from unstructured_ingest.v2.interfaces.process import BaseProcess
-from unstructured_ingest.v2.logger import logger
-from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 
 class UncompressConfig(BaseModel):

unstructured_ingest/{v2/processes → processes}/utils/blob_storage.py

@@ -2,9 +2,9 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any
 
+from unstructured_ingest.data_types.file_data import FileData
+from unstructured_ingest.interfaces import UploadStager, UploadStagerConfig
 from unstructured_ingest.utils.data_prep import get_data, write_data
-from unstructured_ingest.v2.interfaces import UploadStager, UploadStagerConfig
-from unstructured_ingest.v2.types.file_data import FileData
 
 
 class BlobStoreUploadStagerConfig(UploadStagerConfig):

unstructured_ingest/{v2/unstructured_api.py → unstructured_api.py}

@@ -2,8 +2,8 @@ from dataclasses import fields
 from pathlib import Path
 from typing import TYPE_CHECKING, Optional
 
-from unstructured_ingest.v2.errors import ProviderError, QuotaError, UserAuthError, UserError
-from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.errors_v2 import ProviderError, QuotaError, UserAuthError, UserError
+from unstructured_ingest.logger import logger
 
 if TYPE_CHECKING:
     from unstructured_client.models.operations import PartitionRequest

unstructured_ingest/utils/compression.py

@@ -1,19 +1,10 @@
-import copy
 import os
 import sys
 import tarfile
 import zipfile
-from dataclasses import dataclass
 from pathlib import Path
-from typing import List, Optional
+from typing import Optional
 
-from unstructured_ingest.connector.local import LocalSourceConnector, SimpleLocalConfig
-from unstructured_ingest.interfaces import (
-    BaseConnectorConfig,
-    BaseSingleIngestDoc,
-    ProcessorConfig,
-    ReadConfig,
-)
 from unstructured_ingest.logger import logger
 
 ZIP_FILE_EXT = [".zip"]
@@ -78,41 +69,3 @@ def uncompress_tar_file(tar_filename: str, path: Optional[str] = None) -> str:
             )
         tfile.extractall(path=path)
     return path
-
-
-@dataclass
-class CompressionSourceConnectorMixin:
-    processor_config: ProcessorConfig
-    read_config: ReadConfig
-    connector_config: BaseConnectorConfig
-
-    def process_compressed_doc(self, doc: BaseSingleIngestDoc) -> List[BaseSingleIngestDoc]:
-        """
-        Utility function which helps process compressed files. Extracts the contents and returns
-        generated ingest docs via local source connector
-        """
-        # Download the raw file to local
-        doc.get_file()
-        path = uncompress_file(filename=str(doc.filename))
-        new_read_configs = copy.copy(self.read_config)
-        new_process_configs = copy.copy(self.processor_config)
-        relative_path = path.replace(self.read_config.download_dir, "")
-
-        if self.processor_config.output_dir.endswith(os.sep):
-            new_process_configs.output_dir = f"{self.processor_config.output_dir}{relative_path}"
-        else:
-            new_process_configs.output_dir = (
-                f"{self.processor_config.output_dir}{os.sep}{relative_path}"
-            )
-
-        local_connector = LocalSourceConnector(
-            connector_config=SimpleLocalConfig(
-                input_path=path,
-                recursive=True,
-            ),
-            read_config=new_read_configs,
-            processor_config=new_process_configs,
-        )
-        logger.info(f"created local source connector: {local_connector.to_json()}")
-        local_connector.initialize()
-        return local_connector.get_ingest_docs()

unstructured_ingest/utils/data_prep.py

@@ -3,10 +3,12 @@ import json
 from datetime import datetime
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Generator, Iterable, Optional, Sequence, TypeVar, Union, cast
+from uuid import NAMESPACE_DNS, uuid5
 
+from unstructured_ingest.data_types.file_data import FileData
+from unstructured_ingest.logger import logger
 from unstructured_ingest.utils import ndjson
 from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.logger import logger
 
 if TYPE_CHECKING:
     from pandas import DataFrame
@@ -230,3 +232,9 @@ def get_data_df(path: Path) -> "DataFrame":
             return df
         else:
             raise ValueError(f"Unsupported file type: {path}")
+
+
+def get_enhanced_element_id(element_dict: dict, file_data: FileData) -> str:
+    element_id = element_dict.get("element_id")
+    new_data = f"{element_id}{file_data.identifier}"
+    return str(uuid5(NAMESPACE_DNS, new_data))

unstructured_ingest/utils/html.py

@@ -6,10 +6,10 @@ from uuid import NAMESPACE_DNS, uuid5
 
 from pydantic import BaseModel, Field
 
+from unstructured_ingest.data_types.file_data import FileData, SourceIdentifiers
+from unstructured_ingest.interfaces import DownloadResponse
+from unstructured_ingest.logger import logger
 from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.interfaces import DownloadResponse
-from unstructured_ingest.v2.logger import logger
-from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 if TYPE_CHECKING:
     from bs4.element import Tag

unstructured_ingest/{v2/utils.py → utils/pydantic_models.py}

@@ -3,13 +3,10 @@ from datetime import datetime
 from inspect import isclass
 from pathlib import Path
 from typing import Any
-from uuid import NAMESPACE_DNS, uuid5
 
 from pydantic import BaseModel
 from pydantic.types import _SecretBase
 
-from unstructured_ingest.v2.types.file_data import FileData
-
 
 def is_secret(value: Any) -> bool:
     # Case Secret[int]
@@ -53,9 +50,3 @@ def serialize_base_model_json(model: BaseModel, **json_kwargs) -> str:
 
     # Support json dumps kwargs such as sort_keys
     return json.dumps(model_dict, default=json_serial, **json_kwargs)
-
-
-def get_enhanced_element_id(element_dict: dict, file_data: FileData) -> str:
-    element_id = element_dict.get("element_id")
-    new_data = f"{element_id}{file_data.identifier}"
-    return str(uuid5(NAMESPACE_DNS, new_data))

unstructured_ingest/utils/string_and_date_utils.py

@@ -5,7 +5,7 @@ from datetime import datetime
 
 from dateutil import parser
 
-from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.logger import logger
 
 
 def json_to_dict(json_string: str) -> t.Union[str, t.Dict[str, t.Any]]: