unstructured-ingest 1.0.33__py3-none-any.whl → 1.0.35__py3-none-any.whl

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "1.0.33"  # pragma: no cover
+__version__ = "1.0.35"  # pragma: no cover

unstructured_ingest/processes/connectors/confluence.py

@@ -12,6 +12,7 @@ from unstructured_ingest.data_types.file_data import (
     SourceIdentifiers,
 )
 from unstructured_ingest.error import SourceConnectionError
+from unstructured_ingest.errors_v2 import UserAuthError, UserError
 from unstructured_ingest.interfaces import (
     AccessConfig,
     ConnectionConfig,
@@ -96,7 +97,7 @@ class ConfluenceConnectionConfig(ConnectionConfig):
 
     @requires_dependencies(["atlassian"], extras="confluence")
     @contextmanager
-    def get_client(self) -> "Confluence":
+    def get_client(self) -> Generator["Confluence", None, None]:
         from atlassian import Confluence
 
         access_configs = self.access_config.get_secret_value()
@@ -126,15 +127,36 @@ class ConfluenceIndexer(Indexer):
 
     def precheck(self) -> bool:
         try:
-            # Attempt to retrieve a list of spaces with limit=1.
-            # This should only succeed if all creds are valid
-            with self.connection_config.get_client() as client:
+            self.connection_config.get_client()
+        except Exception as e:
+            logger.exception(f"Failed to connect to Confluence: {e}")
+            raise UserAuthError(f"Failed to connect to Confluence: {e}")
+
+        with self.connection_config.get_client() as client:
+            # opportunistically check the first space in list of all spaces
+            try:
                 client.get_all_spaces(limit=1)
+            except Exception as e:
+                logger.exception(f"Failed to connect to find any Confluence space: {e}")
+                raise UserError(f"Failed to connect to find any Confluence space: {e}")
+
             logger.info("Connection to Confluence successful.")
-            return True
-        except Exception as e:
-            logger.error(f"Failed to connect to Confluence: {e}", exc_info=True)
-            raise SourceConnectionError(f"Failed to connect to Confluence: {e}")
+
+            # If specific spaces are provided, check if we can access them
+            errors = []
+
+            if self.index_config.spaces:
+                for space_key in self.index_config.spaces:
+                    try:
+                        client.get_space(space_key)
+                    except Exception as e:
+                        logger.exception(f"Failed to connect to Confluence: {e}")
+                        errors.append(f"Failed to connect to '{space_key}' space, cause: '{e}'")
+
+            if errors:
+                raise UserError("\n".join(errors))
+
+        return True
 
     def _get_space_ids_and_keys(self) -> List[Tuple[str, int]]:
         """
@@ -406,7 +428,7 @@ class ConfluenceDownloader(Downloader):
                     expand="history.lastUpdated,version,body.view",
                 )
         except Exception as e:
-            logger.error(f"Failed to retrieve page with ID {doc_id}: {e}", exc_info=True)
+            logger.exception(f"Failed to retrieve page with ID {doc_id}: {e}")
             raise SourceConnectionError(f"Failed to retrieve page with ID {doc_id}: {e}")
 
         if not page:

unstructured_ingest/processes/connectors/databricks/volumes.py

@@ -196,9 +196,10 @@ class DatabricksVolumesUploader(Uploader, ABC):
     connection_config: DatabricksVolumesConnectionConfig
 
     def get_output_path(self, file_data: FileData) -> str:
-        if file_data.source_identifiers.fullpath:
+        if file_data.source_identifiers.relative_path:
             return os.path.join(
-                self.upload_config.path, f"{file_data.source_identifiers.fullpath}.json"
+                self.upload_config.path,
+                f"{file_data.source_identifiers.relative_path.lstrip('/')}.json",
             )
         else:
             return os.path.join(

unstructured_ingest/processes/connectors/fsspec/fsspec.py

@@ -343,9 +343,10 @@ class FsspecUploader(Uploader):
             raise self.wrap_error(e=e)
 
     def get_upload_path(self, file_data: FileData) -> Path:
-        upload_path = Path(
-            self.upload_config.path_without_protocol
-        ) / file_data.source_identifiers.fullpath.lstrip("/")
+        upload_path = (
+            Path(self.upload_config.path_without_protocol)
+            / file_data.source_identifiers.relative_path.lstrip("/")
+        )
         updated_upload_path = upload_path.parent / f"{upload_path.name}.json"
         return updated_upload_path
 

unstructured_ingest/processes/connectors/onedrive.py

@@ -369,15 +369,14 @@ class OnedriveUploader(Uploader):
 
         # Use the remote_url from upload_config as the base destination folder
         base_destination_folder = self.upload_config.url
-
-        # Use the file's full path to maintain directory structure, if needed
-        if file_data.source_identifiers and file_data.source_identifiers.fullpath:
-            # Combine the base destination folder with the file's full path
+        # Use the file's relative path to maintain directory structure, if needed
+        if file_data.source_identifiers and file_data.source_identifiers.relative_path:
+            # Combine the base destination folder with the file's relative path
             destination_path = Path(base_destination_folder) / Path(
-                f"{file_data.source_identifiers.fullpath}.json"
+                f"{file_data.source_identifiers.relative_path}.json"
             )
         else:
-            # If no full path is provided, upload directly to the base destination folder
+            # If no relative path is provided, upload directly to the base destination folder
             destination_path = Path(base_destination_folder) / f"{path.name}.json"
 
         destination_folder = destination_path.parent

{unstructured_ingest-1.0.33.dist-info → unstructured_ingest-1.0.35.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: unstructured_ingest
-Version: 1.0.33
+Version: 1.0.35
 Summary: Local ETL data pipeline to get data RAG ready
 Author-email: Unstructured Technologies <devops@unstructuredai.io>
 License-Expression: Apache-2.0

{unstructured_ingest-1.0.33.dist-info → unstructured_ingest-1.0.35.dist-info}/RECORD

@@ -1,5 +1,5 @@
 unstructured_ingest/__init__.py,sha256=U4S_2y3zgLZVfMenHRaJFBW8yqh2mUBuI291LGQVOJ8,35
-unstructured_ingest/__version__.py,sha256=moV2VeZrrB_QVKOvny6NjEoowwTiGToZWfDpKig5QOQ,43
+unstructured_ingest/__version__.py,sha256=b3qTfBCIVt9b6BKAiKwjS-S1JYzG3JrNJz133p_CIH8,43
 unstructured_ingest/error.py,sha256=qDncnJgbf5ils956RcO2CGlAKYDT5OaEM9Clv1JVTNc,1448
 unstructured_ingest/errors_v2.py,sha256=9RuRCi7lbDxCguDz07y5RiHoQiFIOWwOD7xqzJ2B3Yw,436
 unstructured_ingest/logger.py,sha256=7e_7UeK6hVOd5BQ6i9NzRUAPCS_DF839Y8TjUDywraY,1428
@@ -66,7 +66,7 @@ unstructured_ingest/processes/connectors/airtable.py,sha256=smx5qBSUKwM8V6Xcc7ik
 unstructured_ingest/processes/connectors/astradb.py,sha256=Ob9wQgDxa6BXDPZBOqooNKQgvjIZcMwIe4fW3VlI7h8,18929
 unstructured_ingest/processes/connectors/azure_ai_search.py,sha256=szhSRXzUHk0DE2hGFfjGc_jNFzlUwiRlCtIkuu7tmnk,11524
 unstructured_ingest/processes/connectors/chroma.py,sha256=q5_Fu4xb6_W_NyrPxVa3-jVwZLqVdlBNlR4dFvbd7l0,7235
-unstructured_ingest/processes/connectors/confluence.py,sha256=1oT4A83jSOWR8u8kldHImOBqSLxctdlsR-AZpzJfO9w,21098
+unstructured_ingest/processes/connectors/confluence.py,sha256=C62LVwZYk7H8RfiPb0mbxig2osW5u7KvHIlz4qOJU-0,21954
 unstructured_ingest/processes/connectors/couchbase.py,sha256=KCHoYDNya9B05NIB5D78zXoizFyfpJRepcYBe1nLSOs,12298
 unstructured_ingest/processes/connectors/delta_table.py,sha256=2DFox_Vzoopt_D3Jy3rCjrrTGMutG2INIrwCeoIohRY,7340
 unstructured_ingest/processes/connectors/discord.py,sha256=6yEJ_agfKUqsV43wFsbMkcd8lcLJC0uqbo4izjdZ3rU,5294
@@ -79,7 +79,7 @@ unstructured_ingest/processes/connectors/local.py,sha256=LluTLKv4g7FbJb4A6vuSxI9
 unstructured_ingest/processes/connectors/milvus.py,sha256=Jr9cul7By03tGAPFnFBoqncnNWwbhKd-qbmkuqnin8U,8908
 unstructured_ingest/processes/connectors/mongodb.py,sha256=1g_5bfbS6lah3nsOXqLAanR3zNYJ47_Njw_uV-uj3_U,14324
 unstructured_ingest/processes/connectors/neo4j.py,sha256=ztxvI9KY8RF5kYUuMGSzzN5mz7Fu_4Ai9P7dqCpJLc0,20267
-unstructured_ingest/processes/connectors/onedrive.py,sha256=k0bhQCCSIgmHAk3lQd4CMA3dc4fPAjegNlLxlDWGowc,19284
+unstructured_ingest/processes/connectors/onedrive.py,sha256=JIADpc31PI9Yzr0raF6bSqzes2jhfcniUzew1aKVWeI,19305
 unstructured_ingest/processes/connectors/outlook.py,sha256=zHM5frO7CqQG0-KcTyX49aZeSlsvVrl8kh_lR_ESgQw,9275
 unstructured_ingest/processes/connectors/pinecone.py,sha256=pSREUNsQqel6q1EFZsFWelg-uZgGubQY5m_6nVnBFKs,15090
 unstructured_ingest/processes/connectors/redisdb.py,sha256=rTihbfv0Mlk1eo5Izn-JXRu5Ad5C-KD58nSqeKsaZJ8,8024
@@ -92,7 +92,7 @@ unstructured_ingest/processes/connectors/assets/__init__.py,sha256=47DEQpj8HBSa-
 unstructured_ingest/processes/connectors/assets/databricks_delta_table_schema.sql,sha256=8a9HTcRWA6IuswSD632b_uZSO6Dax_0rUYnflqktcek,226
 unstructured_ingest/processes/connectors/assets/weaviate_collection_config.json,sha256=SJlIO0kXxy866tWQ8bEzvwLwflsoUMIS-OKlxMvHIuE,504
 unstructured_ingest/processes/connectors/databricks/__init__.py,sha256=RtKAPyNtXh6fzEsOQ08pA0-vC1uMr3KqYG6cqiBoo70,2133
-unstructured_ingest/processes/connectors/databricks/volumes.py,sha256=fZeXRozTUM3JeZlmsxhn_glqRhxr8CGG-8I8QRhRcP8,8232
+unstructured_ingest/processes/connectors/databricks/volumes.py,sha256=yT5JFbVzAEOJsKjfGH8KG3eQfKaTNFEsg_FVDPVK7Xs,8271
 unstructured_ingest/processes/connectors/databricks/volumes_aws.py,sha256=WhGTp6aRTLSdc4GChCL4mz2b-IanderW8j1IqezX6YA,2958
 unstructured_ingest/processes/connectors/databricks/volumes_azure.py,sha256=pF2d6uAIbwJJUeOIG5xknUMCGc5d9Aztmc2776wp-a0,3740
 unstructured_ingest/processes/connectors/databricks/volumes_gcp.py,sha256=y9AvVl6PtnIxlTlrPj_wyHBDBRJNq3uoTOuZwTryNg8,2994
@@ -109,7 +109,7 @@ unstructured_ingest/processes/connectors/fsspec/__init__.py,sha256=3HTdw4L4mdN4W
 unstructured_ingest/processes/connectors/fsspec/azure.py,sha256=31VNiG5YnXfhrFX7QJ2O1ubeWHxbe1sYVIztefbscAQ,7148
 unstructured_ingest/processes/connectors/fsspec/box.py,sha256=1gLS7xR2vbjgKBrQ4ZpI1fKTsJuIDfXuAzx_a4FzxG4,5873
 unstructured_ingest/processes/connectors/fsspec/dropbox.py,sha256=HwwKjQmjM7yFk9Esh_F20xDisRPXGUkFduzaasByRDE,8355
-unstructured_ingest/processes/connectors/fsspec/fsspec.py,sha256=4K8Q2D_6_HCqTVM3HBJv3SNz9gjbQhk44nzeSheDpzA,14462
+unstructured_ingest/processes/connectors/fsspec/fsspec.py,sha256=d_ig69_tuSWczwPxzZue1xTYMYqYqUe-dg1jMEjC8M0,14481
 unstructured_ingest/processes/connectors/fsspec/gcs.py,sha256=ouxISCKpZTAj3T6pWGYbASu93wytJjl5WSICvQcrgfE,7172
 unstructured_ingest/processes/connectors/fsspec/s3.py,sha256=2ZV6b2E2pIsf_ab1Lty74FwpMnJZhpQUdamPgpwcKsQ,7141
 unstructured_ingest/processes/connectors/fsspec/sftp.py,sha256=pR_a2SgLjt8ffNkariHrPB1E0HVSTj5h3pt7KxTU3TI,6371
@@ -231,8 +231,8 @@ unstructured_ingest/utils/ndjson.py,sha256=nz8VUOPEgAFdhaDOpuveknvCU4x82fVwqE01q
 unstructured_ingest/utils/pydantic_models.py,sha256=BT_j15e4rX40wQbt8LUXbqfPhA3rJn1PHTI_G_A_EHY,1720
 unstructured_ingest/utils/string_and_date_utils.py,sha256=oXOI6rxXq-8ncbk7EoJK0WCcTXWj75EzKl8pfQMID3U,2522
 unstructured_ingest/utils/table.py,sha256=WZechczgVFvlodUWFcsnCGvBNh1xRm6hr0VbJTPxKAc,3669
-unstructured_ingest-1.0.33.dist-info/METADATA,sha256=d8F0hFb3s-aLloV1TGFLDHRa8CHwuTduHFS1neEHu6s,8747
-unstructured_ingest-1.0.33.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-unstructured_ingest-1.0.33.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
-unstructured_ingest-1.0.33.dist-info/licenses/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
-unstructured_ingest-1.0.33.dist-info/RECORD,,
+unstructured_ingest-1.0.35.dist-info/METADATA,sha256=rpeaT-RpY6IFgDR7tIIxylJL1t-geRuhk3QKQq6JSDY,8747
+unstructured_ingest-1.0.35.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+unstructured_ingest-1.0.35.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
+unstructured_ingest-1.0.35.dist-info/licenses/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
+unstructured_ingest-1.0.35.dist-info/RECORD,,