PyPI - unstructured-ingest - Versions diffs - 0.4.1__py3-none-any.whl → 0.4.3__py3-none-any.whl - Mend

unstructured-ingest 0.4.1py3-none-any.whl → 0.4.3py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of unstructured-ingest might be problematic. Click here for more details.

Files changed (45) hide show

unstructured_ingest/v2/processes/connectors/confluence.py CHANGED Viewed

@@ -7,6 +7,7 @@ from pydantic import Field, Secret
 from unstructured_ingest.error import SourceConnectionError
 from unstructured_ingest.utils.dep_check import requires_dependencies
+from unstructured_ingest.utils.html import HtmlMixin
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
     ConnectionConfig,
@@ -180,19 +181,8 @@ class ConfluenceIndexer(Indexer):
                 yield file_data
-class ConfluenceDownloaderConfig(DownloaderConfig):
-    extract_images: bool = Field(
-        default=False,
-        description="if true, will download images and replace "
-        "the html content with base64 encoded images",
-    )
-    extract_files: bool = Field(
-        default=False, description="if true, will download any embedded files"
-    )
-    force_download: bool = Field(
-        default=False,
-        description="if true, will redownload extracted files even if they already exist locally",
-    )
+class ConfluenceDownloaderConfig(DownloaderConfig, HtmlMixin):
+    pass
 @dataclass
@@ -206,24 +196,27 @@ class ConfluenceDownloader(Downloader):
     ) -> list[DownloadResponse]:
         if not self.download_config.extract_files:
             return []
-        from unstructured_ingest.utils.html import download_embedded_files
+        url = current_file_data.metadata.url
+        if url is None:
+            logger.warning(
+                f"""Missing URL for file: {current_file_data.source_identifiers.filename}.
+                Skipping file extraction."""
+            )
+            return []
         filepath = current_file_data.source_identifiers.relative_path
         download_path = Path(self.download_dir) / filepath
         download_dir = download_path.with_suffix("")
-        return download_embedded_files(
+        return self.download_config.extract_embedded_files(
+            url=url,
             download_dir=download_dir,
             original_filedata=current_file_data,
-            original_html=html,
+            html=html,
             session=session,
-            force_download=self.download_config.force_download,
         )
     def run(self, file_data: FileData, **kwargs) -> download_responses:
         from bs4 import BeautifulSoup
-        from unstructured_ingest.utils.html import convert_image_tags
         doc_id = file_data.identifier
         try:
             with self.connection_config.get_client() as client:
@@ -246,8 +239,8 @@ class ConfluenceDownloader(Downloader):
         content = f"<body class='Document' >{title_html}{content}</body>"
         if self.download_config.extract_images:
             with self.connection_config.get_client() as client:
-                content = convert_image_tags(
-                    url=file_data.metadata.url, original_html=content, session=client._session
+                content = self.download_config.extract_html_images(
+                    url=file_data.metadata.url, html=content, session=client._session
                 )
         filepath = file_data.source_identifiers.relative_path

unstructured_ingest/v2/processes/connectors/databricks/volumes_table.py CHANGED Viewed

@@ -3,10 +3,11 @@ import os
 from contextlib import contextmanager
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Any, Generator, Optional
+from typing import Any, Generator
 from pydantic import Field
+from unstructured_ingest.utils.data_prep import write_data
 from unstructured_ingest.v2.interfaces import FileData, Uploader, UploaderConfig
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connector_registry import (
@@ -14,9 +15,9 @@ from unstructured_ingest.v2.processes.connector_registry import (
 )
 from unstructured_ingest.v2.processes.connectors.databricks.volumes import DatabricksPathMixin
 from unstructured_ingest.v2.processes.connectors.sql.databricks_delta_tables import (
-    DatabrickDeltaTablesConnectionConfig,
-    DatabrickDeltaTablesUploadStager,
-    DatabrickDeltaTablesUploadStagerConfig,
+    DatabricksDeltaTablesConnectionConfig,
+    DatabricksDeltaTablesUploadStager,
+    DatabricksDeltaTablesUploadStagerConfig,
 )
 CONNECTOR_TYPE = "databricks_volume_delta_tables"
@@ -28,17 +29,16 @@ class DatabricksVolumeDeltaTableUploaderConfig(UploaderConfig, DatabricksPathMix
 @dataclass
-class DatabricksVolumeDeltaTableStager(DatabrickDeltaTablesUploadStager):
-    def write_output(self, output_path: Path, data: list[dict], indent: Optional[int] = 2) -> None:
+class DatabricksVolumeDeltaTableStager(DatabricksDeltaTablesUploadStager):
+    def write_output(self, output_path: Path, data: list[dict]) -> None:
         # To avoid new line issues when migrating from volumes into delta tables, omit indenting
         # and always write it as a json file
-        with output_path.with_suffix(".json").open("w") as f:
-            json.dump(data, f)
+        write_data(path=output_path.with_suffix(".json"), data=data, indent=None)
 @dataclass
 class DatabricksVolumeDeltaTableUploader(Uploader):
-    connection_config: DatabrickDeltaTablesConnectionConfig
+    connection_config: DatabricksDeltaTablesConnectionConfig
     upload_config: DatabricksVolumeDeltaTableUploaderConfig
     connector_type: str = CONNECTOR_TYPE
@@ -78,7 +78,10 @@ class DatabricksVolumeDeltaTableUploader(Uploader):
     @contextmanager
     def get_cursor(self, **connect_kwargs) -> Generator[Any, None, None]:
         with self.connection_config.get_cursor(**connect_kwargs) as cursor:
+            logger.debug(f"executing: USE CATALOG: '{self.upload_config.catalog}'")
             cursor.execute(f"USE CATALOG '{self.upload_config.catalog}'")
+            logger.debug(f"executing: USE DATABASE: {self.upload_config.database}")
+            cursor.execute(f"USE DATABASE {self.upload_config.database}")
             yield cursor
     def run(self, path: Path, file_data: FileData, **kwargs: Any) -> None:
@@ -98,9 +101,9 @@ class DatabricksVolumeDeltaTableUploader(Uploader):
 databricks_volumes_delta_tables_destination_entry = DestinationRegistryEntry(
-    connection_config=DatabrickDeltaTablesConnectionConfig,
+    connection_config=DatabricksDeltaTablesConnectionConfig,
     uploader=DatabricksVolumeDeltaTableUploader,
     uploader_config=DatabricksVolumeDeltaTableUploaderConfig,
     upload_stager=DatabricksVolumeDeltaTableStager,
-    upload_stager_config=DatabrickDeltaTablesUploadStagerConfig,
+    upload_stager_config=DatabricksDeltaTablesUploadStagerConfig,
 )

unstructured_ingest/v2/processes/connectors/sql/databricks_delta_tables.py CHANGED Viewed

@@ -31,7 +31,7 @@ if TYPE_CHECKING:
 CONNECTOR_TYPE = "databricks_delta_tables"
-class DatabrickDeltaTablesAccessConfig(SQLAccessConfig):
+class DatabricksDeltaTablesAccessConfig(SQLAccessConfig):
     token: Optional[str] = Field(default=None, description="Databricks Personal Access Token")
     client_id: Optional[str] = Field(default=None, description="Client ID of the OAuth app.")
     client_secret: Optional[str] = Field(
@@ -39,8 +39,8 @@ class DatabrickDeltaTablesAccessConfig(SQLAccessConfig):
     )
-class DatabrickDeltaTablesConnectionConfig(SQLConnectionConfig):
-    access_config: Secret[DatabrickDeltaTablesAccessConfig]
+class DatabricksDeltaTablesConnectionConfig(SQLConnectionConfig):
+    access_config: Secret[DatabricksDeltaTablesAccessConfig]
     server_hostname: str = Field(description="server hostname connection config value")
     http_path: str = Field(description="http path connection config value")
     user_agent: str = "unstructuredio_oss"
@@ -102,24 +102,24 @@ class DatabrickDeltaTablesConnectionConfig(SQLConnectionConfig):
             yield cursor
-class DatabrickDeltaTablesUploadStagerConfig(SQLUploadStagerConfig):
+class DatabricksDeltaTablesUploadStagerConfig(SQLUploadStagerConfig):
     pass
-class DatabrickDeltaTablesUploadStager(SQLUploadStager):
-    upload_stager_config: DatabrickDeltaTablesUploadStagerConfig
+class DatabricksDeltaTablesUploadStager(SQLUploadStager):
+    upload_stager_config: DatabricksDeltaTablesUploadStagerConfig
-class DatabrickDeltaTablesUploaderConfig(SQLUploaderConfig):
+class DatabricksDeltaTablesUploaderConfig(SQLUploaderConfig):
     catalog: str = Field(description="Name of the catalog in the Databricks Unity Catalog service")
     database: str = Field(description="Database name", default="default")
     table_name: str = Field(description="Table name")
 @dataclass
-class DatabrickDeltaTablesUploader(SQLUploader):
-    upload_config: DatabrickDeltaTablesUploaderConfig
-    connection_config: DatabrickDeltaTablesConnectionConfig
+class DatabricksDeltaTablesUploader(SQLUploader):
+    upload_config: DatabricksDeltaTablesUploaderConfig
+    connection_config: DatabricksDeltaTablesConnectionConfig
     connector_type: str = CONNECTOR_TYPE
     @contextmanager
@@ -205,9 +205,9 @@ class DatabrickDeltaTablesUploader(SQLUploader):
 databricks_delta_tables_destination_entry = DestinationRegistryEntry(
-    connection_config=DatabrickDeltaTablesConnectionConfig,
-    uploader=DatabrickDeltaTablesUploader,
-    uploader_config=DatabrickDeltaTablesUploaderConfig,
-    upload_stager=DatabrickDeltaTablesUploadStager,
-    upload_stager_config=DatabrickDeltaTablesUploadStagerConfig,
+    connection_config=DatabricksDeltaTablesConnectionConfig,
+    uploader=DatabricksDeltaTablesUploader,
+    uploader_config=DatabricksDeltaTablesUploaderConfig,
+    upload_stager=DatabricksDeltaTablesUploadStager,
+    upload_stager_config=DatabricksDeltaTablesUploadStagerConfig,
 )

unstructured_ingest/v2/processes/connectors/sql/sql.py CHANGED Viewed

@@ -292,6 +292,9 @@ class SQLUploadStager(UploadStager):
             df[column] = df[column].apply(str)
         return df
+    def write_output(self, output_path: Path, data: list[dict]) -> None:
+        write_data(path=output_path, data=data)
     def run(
         self,
         elements_filepath: Path,
@@ -314,7 +317,7 @@ class SQLUploadStager(UploadStager):
         output_filename = f"{Path(output_filename).stem}{output_filename_suffix}"
         output_path = self.get_output_path(output_filename=output_filename, output_dir=output_dir)
-        write_data(path=output_path, data=df.to_dict(orient="records"))
+        self.write_output(output_path=output_path, data=df.to_dict(orient="records"))
         return output_path

unstructured_ingest/v2/processes/connectors/weaviate/weaviate.py CHANGED Viewed

@@ -3,6 +3,7 @@ from abc import ABC, abstractmethod
 from contextlib import contextmanager
 from dataclasses import dataclass, field
 from datetime import date, datetime
+from pathlib import Path
 from typing import TYPE_CHECKING, Any, Generator, Optional
 from dateutil import parser
@@ -15,10 +16,10 @@ from unstructured_ingest.v2.interfaces import (
     AccessConfig,
     ConnectionConfig,
     FileData,
-    Uploader,
     UploaderConfig,
     UploadStager,
     UploadStagerConfig,
+    VectorDBUploader,
 )
 from unstructured_ingest.v2.logger import logger
@@ -160,7 +161,9 @@ class WeaviateUploadStager(UploadStager):
 class WeaviateUploaderConfig(UploaderConfig):
-    collection: str = Field(description="The name of the collection this object belongs to")
+    collection: Optional[str] = Field(
+        description="The name of the collection this object belongs to", default=None
+    )
     batch_size: Optional[int] = Field(default=None, description="Number of records per batch")
     requests_per_minute: Optional[int] = Field(default=None, description="Rate limit for upload")
     dynamic_batch: bool = Field(default=True, description="Whether to use dynamic batch")
@@ -205,17 +208,50 @@ class WeaviateUploaderConfig(UploaderConfig):
 @dataclass
-class WeaviateUploader(Uploader, ABC):
+class WeaviateUploader(VectorDBUploader, ABC):
     upload_config: WeaviateUploaderConfig
     connection_config: WeaviateConnectionConfig
+    def _collection_exists(self, collection_name: Optional[str] = None):
+        collection_name = collection_name or self.upload_config.collection
+        with self.connection_config.get_client() as weaviate_client:
+            return weaviate_client.collections.exists(name=collection_name)
     def precheck(self) -> None:
         try:
             self.connection_config.get_client()
+            # only if collection name populated should we check that it exists
+            if self.upload_config.collection and not self._collection_exists():
+                raise DestinationConnectionError(
+                    f"collection '{self.upload_config.collection}' does not exist"
+                )
         except Exception as e:
             logger.error(f"Failed to validate connection {e}", exc_info=True)
             raise DestinationConnectionError(f"failed to validate connection: {e}")
+    def init(self, *kwargs: Any) -> None:
+        self.create_destination()
+    def create_destination(
+        self, destination_name: str = "elements", vector_length: Optional[int] = None, **kwargs: Any
+    ) -> bool:
+        collection_name = self.upload_config.collection or destination_name
+        self.upload_config.collection = collection_name
+        connectors_dir = Path(__file__).parents[1]
+        collection_config_file = connectors_dir / "assets" / "weaviate_collection_config.json"
+        with collection_config_file.open() as f:
+            collection_config = json.load(f)
+        collection_config["class"] = collection_name
+        if not self._collection_exists():
+            logger.info(
+                f"creating default weaviate collection '{collection_name}' with default configs"
+            )
+            with self.connection_config.get_client() as weaviate_client:
+                weaviate_client.collections.create_from_dict(config=collection_config)
+                return True
+        logger.debug(f"collection with name '{collection_name}' already exists, skipping creation")
+        return False
     def check_for_errors(self, client: "WeaviateClient") -> None:
         failed_uploads = client.batch.failed_objects
         if failed_uploads:
@@ -253,6 +289,8 @@ class WeaviateUploader(Uploader, ABC):
             f"writing {len(data)} objects to destination "
             f"class {self.connection_config.access_config} "
         )
+        if not self.upload_config.collection:
+            raise ValueError("No collection specified")
         with self.connection_config.get_client() as weaviate_client:
             self.delete_by_record_id(client=weaviate_client, file_data=file_data)

unstructured_ingest/v2/processes/embedder.py CHANGED Viewed

@@ -184,6 +184,9 @@ class EmbedderConfig(BaseModel):
 class Embedder(BaseProcess, ABC):
     config: EmbedderConfig
+    def init(self, *kwargs: Any) -> None:
+        self.config.get_embedder().initialize()
     def run(self, elements_filepath: Path, **kwargs: Any) -> list[dict]:
         # TODO update base embedder classes to support async
         embedder = self.config.get_embedder()

{unstructured_ingest-0.4.1.dist-info → unstructured_ingest-0.4.3.dist-info}/METADATA RENAMED Viewed

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unstructured-ingest
-Version: 0.4.1
+Version: 0.4.3
 Summary: A library that prepares raw documents for downstream ML tasks.
 Home-page: https://github.com/Unstructured-IO/unstructured-ingest
 Author: Unstructured Technologies
@@ -22,38 +22,38 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.9.0,<3.14
 Description-Content-Type: text/markdown
 License-File: LICENSE.md
-Requires-Dist: tqdm
-Requires-Dist: pydantic>=2.7
-Requires-Dist: python-dateutil
-Requires-Dist: pandas
 Requires-Dist: dataclasses-json
+Requires-Dist: pandas
+Requires-Dist: python-dateutil
 Requires-Dist: opentelemetry-sdk
 Requires-Dist: click
+Requires-Dist: pydantic>=2.7
+Requires-Dist: tqdm
 Provides-Extra: airtable
 Requires-Dist: pyairtable; extra == "airtable"
 Provides-Extra: astradb
 Requires-Dist: astrapy; extra == "astradb"
 Provides-Extra: azure
-Requires-Dist: adlfs; extra == "azure"
 Requires-Dist: fsspec; extra == "azure"
+Requires-Dist: adlfs; extra == "azure"
 Provides-Extra: azure-ai-search
 Requires-Dist: azure-search-documents; extra == "azure-ai-search"
 Provides-Extra: bedrock
-Requires-Dist: aioboto3; extra == "bedrock"
 Requires-Dist: boto3; extra == "bedrock"
+Requires-Dist: aioboto3; extra == "bedrock"
 Provides-Extra: biomed
 Requires-Dist: bs4; extra == "biomed"
 Requires-Dist: requests; extra == "biomed"
 Provides-Extra: box
-Requires-Dist: boxfs; extra == "box"
 Requires-Dist: fsspec; extra == "box"
+Requires-Dist: boxfs; extra == "box"
 Provides-Extra: chroma
 Requires-Dist: chromadb; extra == "chroma"
 Provides-Extra: clarifai
 Requires-Dist: clarifai; extra == "clarifai"
 Provides-Extra: confluence
-Requires-Dist: atlassian-python-api; extra == "confluence"
 Requires-Dist: requests; extra == "confluence"
+Requires-Dist: atlassian-python-api; extra == "confluence"
 Provides-Extra: couchbase
 Requires-Dist: couchbase; extra == "couchbase"
 Provides-Extra: csv
@@ -63,8 +63,8 @@ Requires-Dist: databricks-sql-connector; extra == "databricks-delta-tables"
 Provides-Extra: databricks-volumes
 Requires-Dist: databricks-sdk; extra == "databricks-volumes"
 Provides-Extra: delta-table
-Requires-Dist: deltalake; extra == "delta-table"
 Requires-Dist: boto3; extra == "delta-table"
+Requires-Dist: deltalake; extra == "delta-table"
 Provides-Extra: discord
 Requires-Dist: discord.py; extra == "discord"
 Provides-Extra: doc
@@ -72,8 +72,8 @@ Requires-Dist: unstructured[docx]; extra == "doc"
 Provides-Extra: docx
 Requires-Dist: unstructured[docx]; extra == "docx"
 Provides-Extra: dropbox
-Requires-Dist: dropboxdrivefs; extra == "dropbox"
 Requires-Dist: fsspec; extra == "dropbox"
+Requires-Dist: dropboxdrivefs; extra == "dropbox"
 Provides-Extra: duckdb
 Requires-Dist: duckdb; extra == "duckdb"
 Provides-Extra: elasticsearch
@@ -93,11 +93,11 @@ Provides-Extra: epub
 Requires-Dist: unstructured[epub]; extra == "epub"
 Provides-Extra: gcs
 Requires-Dist: gcsfs; extra == "gcs"
-Requires-Dist: bs4; extra == "gcs"
 Requires-Dist: fsspec; extra == "gcs"
+Requires-Dist: bs4; extra == "gcs"
 Provides-Extra: github
-Requires-Dist: pygithub>1.58.0; extra == "github"
 Requires-Dist: requests; extra == "github"
+Requires-Dist: pygithub>1.58.0; extra == "github"
 Provides-Extra: gitlab
 Requires-Dist: python-gitlab; extra == "gitlab"
 Provides-Extra: google-drive
@@ -126,16 +126,16 @@ Requires-Dist: neo4j; extra == "neo4j"
 Requires-Dist: networkx; extra == "neo4j"
 Requires-Dist: cymple; extra == "neo4j"
 Provides-Extra: notion
-Requires-Dist: httpx; extra == "notion"
-Requires-Dist: backoff; extra == "notion"
 Requires-Dist: htmlBuilder; extra == "notion"
+Requires-Dist: backoff; extra == "notion"
 Requires-Dist: notion-client; extra == "notion"
+Requires-Dist: httpx; extra == "notion"
 Provides-Extra: odt
 Requires-Dist: unstructured[odt]; extra == "odt"
 Provides-Extra: onedrive
-Requires-Dist: bs4; extra == "onedrive"
-Requires-Dist: Office365-REST-Python-Client; extra == "onedrive"
 Requires-Dist: msal; extra == "onedrive"
+Requires-Dist: Office365-REST-Python-Client; extra == "onedrive"
+Requires-Dist: bs4; extra == "onedrive"
 Provides-Extra: openai
 Requires-Dist: openai; extra == "openai"
 Requires-Dist: tiktoken; extra == "openai"
@@ -144,8 +144,8 @@ Requires-Dist: opensearch-py; extra == "opensearch"
 Provides-Extra: org
 Requires-Dist: unstructured[org]; extra == "org"
 Provides-Extra: outlook
-Requires-Dist: Office365-REST-Python-Client; extra == "outlook"
 Requires-Dist: msal; extra == "outlook"
+Requires-Dist: Office365-REST-Python-Client; extra == "outlook"
 Provides-Extra: pdf
 Requires-Dist: unstructured[pdf]; extra == "pdf"
 Provides-Extra: pinecone
@@ -174,26 +174,26 @@ Requires-Dist: s3fs; extra == "s3"
 Provides-Extra: salesforce
 Requires-Dist: simple-salesforce; extra == "salesforce"
 Provides-Extra: sftp
-Requires-Dist: paramiko; extra == "sftp"
 Requires-Dist: fsspec; extra == "sftp"
+Requires-Dist: paramiko; extra == "sftp"
 Provides-Extra: sharepoint
-Requires-Dist: Office365-REST-Python-Client; extra == "sharepoint"
 Requires-Dist: msal; extra == "sharepoint"
+Requires-Dist: Office365-REST-Python-Client; extra == "sharepoint"
 Provides-Extra: singlestore
 Requires-Dist: singlestoredb; extra == "singlestore"
 Provides-Extra: slack
 Requires-Dist: slack-sdk[optional]; extra == "slack"
 Provides-Extra: snowflake
-Requires-Dist: psycopg2-binary; extra == "snowflake"
 Requires-Dist: snowflake-connector-python; extra == "snowflake"
+Requires-Dist: psycopg2-binary; extra == "snowflake"
 Provides-Extra: togetherai
 Requires-Dist: together; extra == "togetherai"
 Provides-Extra: tsv
 Requires-Dist: unstructured[tsv]; extra == "tsv"
 Provides-Extra: vastdb
-Requires-Dist: pyarrow; extra == "vastdb"
 Requires-Dist: vastdb; extra == "vastdb"
 Requires-Dist: ibis; extra == "vastdb"
+Requires-Dist: pyarrow; extra == "vastdb"
 Provides-Extra: vectara
 Requires-Dist: httpx; extra == "vectara"
 Requires-Dist: requests; extra == "vectara"

unstructured-ingest 0.4.1__py3-none-any.whl → 0.4.3__py3-none-any.whl

Potentially problematic release.

unstructured-ingest 0.4.1py3-none-any.whl → 0.4.3py3-none-any.whl