unstructured-ingest 1.0.4__py3-none-any.whl → 1.0.6.dev0__py3-none-any.whl

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "1.0.4"  # pragma: no cover
+__version__ = "1.0.6-dev0"  # pragma: no cover

unstructured_ingest/data_types/file_data.py

@@ -1,6 +1,6 @@
 import json
 from pathlib import Path
-from typing import Any, Optional, Union
+from typing import Any, Optional
 from uuid import NAMESPACE_DNS, uuid5
 
 from pydantic import BaseModel, Field, ValidationError, field_validator, model_validator
@@ -29,7 +29,7 @@ class FileDataSourceMetadata(BaseModel):
     date_created: Optional[str] = None
     date_modified: Optional[str] = None
     date_processed: Optional[str] = None
-    permissions_data: Union[list[dict[str, Any]], dict[str, Any], None] = None
+    permissions_data: Optional[list[dict[str, Any]]] = None
     filesize_bytes: Optional[int] = None
 
 

unstructured_ingest/processes/connectors/confluence.py

@@ -352,6 +352,7 @@ class ConfluenceDownloader(Downloader):
     def _get_permissions_for_space(self, space_id: int) -> Optional[List[dict]]:
         if space_id in self._permissions_cache:
             self._permissions_cache.move_to_end(space_id)  # mark recent use
+            logger.debug(f"Retrieved cached permissions for space {space_id}")
             return self._permissions_cache[space_id]
         else:
             with self.connection_config.get_client() as client:
@@ -371,22 +372,27 @@ class ConfluenceDownloader(Downloader):
                         self._permissions_cache.popitem(last=False)  # LRU/FIFO eviction
                     self._permissions_cache[space_id] = space_permissions
 
+                    logger.debug(f"Retrieved permissions for space {space_id}")
                     return space_permissions
                 except Exception as e:
                     logger.debug(f"Could not retrieve permissions for space {space_id}: {e}")
                     return None
 
-    def _parse_permissions_for_doc(self, doc_id: str, space_permissions: list) -> Optional[dict]:
+    def _parse_permissions_for_doc(
+        self, doc_id: str, space_permissions: list
+    ) -> Optional[list[dict]]:
         with self.connection_config.get_client() as client:
             try:
                 doc_permissions = client.get_all_restrictions_for_content(content_id=doc_id)
                 parsed_permissions_dict = self.parse_permissions(doc_permissions, space_permissions)
+                parsed_permissions_dict = [{k: v} for k, v in parsed_permissions_dict.items()]
 
             except Exception as e:
                 # skip writing any permission metadata
                 logger.debug(f"Could not retrieve permissions for doc {doc_id}: {e}")
                 return None
 
+        logger.debug(f"normalized permissions generated: {parsed_permissions_dict}")
         return parsed_permissions_dict
 
     def run(self, file_data: FileData, **kwargs) -> download_responses:

unstructured_ingest/processes/connectors/google_drive.py

@@ -54,7 +54,8 @@ class GoogleDriveAccessConfig(AccessConfig):
         default=None, description="Credentials values to use for authentication"
     )
     service_account_key_path: Optional[Path] = Field(
-        default=None, description="File path to credentials values to use for authentication"
+        default=None,
+        description="File path to credentials values to use for authentication",
     )
 
     def model_post_init(self, __context: Any) -> None:
@@ -111,10 +112,9 @@ class GoogleDriveIndexerConfig(IndexerConfig):
     extensions: Optional[list[str]] = None
     recursive: bool = False
 
-    def __post_init__(self):
-        # Strip leading period of extension
+    def model_post_init(self, __context: Any) -> None:
         if self.extensions is not None:
-            self.extensions = [e[1:] if e.startswith(".") else e for e in self.extensions]
+            self.extensions = [e.lstrip(".") for e in self.extensions]
 
 
 @dataclass
@@ -275,7 +275,8 @@ class GoogleDriveIndexer(Indexer):
 
         except Exception as e:
             logger.error(
-                "Failed to validate Google Drive connection during precheck", exc_info=True
+                "Failed to validate Google Drive connection during precheck",
+                exc_info=True,
             )
             raise SourceConnectionError(f"Precheck failed: {e}")
 
@@ -284,17 +285,17 @@ class GoogleDriveIndexer(Indexer):
         return record.get("mimeType") == "application/vnd.google-apps.folder"
 
     @staticmethod
-    def map_file_data(f: dict) -> FileData:
-        file_id = f["id"]
-        filename = f.pop("name")
-        url = f.pop("webContentLink", None)
-        version = f.pop("version", None)
-        permissions = f.pop("permissions", None)
-        date_created_str = f.pop("createdTime", None)
+    def map_file_data(root_info: dict) -> FileData:
+        file_id = root_info["id"]
+        filename = root_info.pop("name")
+        url = root_info.pop("webContentLink", None)
+        version = root_info.pop("version", None)
+        permissions = root_info.pop("permissions", None)
+        date_created_str = root_info.pop("createdTime", None)
         date_created_dt = parser.parse(date_created_str) if date_created_str else None
-        date_modified_str = f.pop("modifiedTime", None)
-        parent_path = f.pop("parent_path", None)
-        parent_root_path = f.pop("parent_root_path", None)
+        date_modified_str = root_info.pop("modifiedTime", None)
+        parent_path = root_info.pop("parent_path", None)
+        parent_root_path = root_info.pop("parent_root_path", None)
         date_modified_dt = parser.parse(date_modified_str) if date_modified_str else None
         if (
             parent_path
@@ -323,7 +324,7 @@ class GoogleDriveIndexer(Indexer):
                     "file_id": file_id,
                 },
             ),
-            additional_metadata=f,
+            additional_metadata=root_info,
         )
 
     def get_paginated_results(
@@ -404,13 +405,14 @@ class GoogleDriveIndexer(Indexer):
             data = []
             for f in file_contents:
                 f["permissions"] = self.extract_permissions(f.get("permissions"))
-                data.append(self.map_file_data(f=f))
+                data.append(self.map_file_data(root_info=f))
         for d in data:
             d.metadata.record_locator["drive_id"]: object_id
         return data
 
-    def extract_permissions(self, permissions: list[dict]) -> dict:
+    def extract_permissions(self, permissions: Optional[list[dict]]) -> list[dict]:
         if not permissions:
+            logger.debug("no permissions found")
             return {}
 
         # https://developers.google.com/workspace/drive/api/guides/ref-roles
@@ -442,7 +444,8 @@ class GoogleDriveIndexer(Indexer):
             for key in role_dict:
                 role_dict[key] = sorted(role_dict[key])
 
-        return normalized_permissions
+        logger.debug(f"normalized permissions generated: {normalized_permissions}")
+        return [{k: v} for k, v in normalized_permissions.items()]
 
     def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
         with self.connection_config.get_client() as client:

{unstructured_ingest-1.0.4.dist-info → unstructured_ingest-1.0.6.dev0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: unstructured_ingest
-Version: 1.0.4
+Version: 1.0.6.dev0
 Summary: Local ETL data pipeline to get data RAG ready
 Author-email: Unstructured Technologies <devops@unstructuredai.io>
 License-Expression: Apache-2.0

{unstructured_ingest-1.0.4.dist-info → unstructured_ingest-1.0.6.dev0.dist-info}/RECORD

@@ -1,5 +1,5 @@
 unstructured_ingest/__init__.py,sha256=U4S_2y3zgLZVfMenHRaJFBW8yqh2mUBuI291LGQVOJ8,35
-unstructured_ingest/__version__.py,sha256=MizK8W2VY6aXUudG1jVogTj7GJ2uwduw5iryFPwi0tM,42
+unstructured_ingest/__version__.py,sha256=f9THPBYTYgzAGHqOPz3K_-VIVaQAVKMJgCpBy7w0j_k,47
 unstructured_ingest/error.py,sha256=qDncnJgbf5ils956RcO2CGlAKYDT5OaEM9Clv1JVTNc,1448
 unstructured_ingest/errors_v2.py,sha256=9RuRCi7lbDxCguDz07y5RiHoQiFIOWwOD7xqzJ2B3Yw,436
 unstructured_ingest/logger.py,sha256=7e_7UeK6hVOd5BQ6i9NzRUAPCS_DF839Y8TjUDywraY,1428
@@ -19,7 +19,7 @@ unstructured_ingest/cli/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5N
 unstructured_ingest/cli/utils/click.py,sha256=1_eJgrwS2DFBl1jZPLsj1vgVgR7agFBIEBe4A_n7mH4,7827
 unstructured_ingest/cli/utils/model_conversion.py,sha256=hMjAfOVvO1RXTDsw26mmersdncvddkb_rP9JTEgVVCw,7649
 unstructured_ingest/data_types/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-unstructured_ingest/data_types/file_data.py,sha256=E-09hkI4ms4yj-g_aQPIrnm0kbiZLwukCnbwp6OpobQ,3859
+unstructured_ingest/data_types/file_data.py,sha256=J0RQa7YXhhxiLVzhPbF5Hl2nzSpxLFK9vrP6RTBWlSg,3833
 unstructured_ingest/embed/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unstructured_ingest/embed/azure_openai.py,sha256=_-I-nwd-wdCiKkSdYBL4UKrTZ2UPWsM_0T69fcObs_I,1707
 unstructured_ingest/embed/bedrock.py,sha256=t58V_QQjWPO62CTuP0aLFMDisPeXpxG2xSFGUhN-JvI,7726
@@ -65,13 +65,13 @@ unstructured_ingest/processes/connectors/airtable.py,sha256=smx5qBSUKwM8V6Xcc7ik
 unstructured_ingest/processes/connectors/astradb.py,sha256=Ob9wQgDxa6BXDPZBOqooNKQgvjIZcMwIe4fW3VlI7h8,18929
 unstructured_ingest/processes/connectors/azure_ai_search.py,sha256=szhSRXzUHk0DE2hGFfjGc_jNFzlUwiRlCtIkuu7tmnk,11524
 unstructured_ingest/processes/connectors/chroma.py,sha256=q5_Fu4xb6_W_NyrPxVa3-jVwZLqVdlBNlR4dFvbd7l0,7235
-unstructured_ingest/processes/connectors/confluence.py,sha256=7uRgmpX3NcVzA2V7VcngzjMQ69pS0J2wu6cbMp7AFA0,20739
+unstructured_ingest/processes/connectors/confluence.py,sha256=1oT4A83jSOWR8u8kldHImOBqSLxctdlsR-AZpzJfO9w,21098
 unstructured_ingest/processes/connectors/couchbase.py,sha256=KCHoYDNya9B05NIB5D78zXoizFyfpJRepcYBe1nLSOs,12298
 unstructured_ingest/processes/connectors/delta_table.py,sha256=2DFox_Vzoopt_D3Jy3rCjrrTGMutG2INIrwCeoIohRY,7340
 unstructured_ingest/processes/connectors/discord.py,sha256=6yEJ_agfKUqsV43wFsbMkcd8lcLJC0uqbo4izjdZ3rU,5294
 unstructured_ingest/processes/connectors/github.py,sha256=smHCz6jOH1p_hW2S25bYunBBj_pYjz8HTw6wkzaJz_A,7765
 unstructured_ingest/processes/connectors/gitlab.py,sha256=6h1CdqznJmzeWxGfXrFLdNdT23PExGnUMMX7usK_4Kk,10013
-unstructured_ingest/processes/connectors/google_drive.py,sha256=mcplAPbQ_A_MIsIXWc7K0YtEXMIMmluefsrzddJQNFw,21674
+unstructured_ingest/processes/connectors/google_drive.py,sha256=YMuobb9AZq-0pp6rbYyXNizs3jA6z3nQjO9cK_nncXQ,21936
 unstructured_ingest/processes/connectors/jira.py,sha256=eG8yTn8ZVEz7rBJ-ha8i_d9hEh6VALN6QJT_vbYvbL0,17142
 unstructured_ingest/processes/connectors/kdbai.py,sha256=XhxYpKSAoFPBsDQWwNuLX03DCxOVr7yquj9VYM55Rtc,5174
 unstructured_ingest/processes/connectors/local.py,sha256=LluTLKv4g7FbJb4A6vuSxI9VhzKZuuQUpDS-cVNAQ2g,7426
@@ -230,8 +230,8 @@ unstructured_ingest/utils/ndjson.py,sha256=nz8VUOPEgAFdhaDOpuveknvCU4x82fVwqE01q
 unstructured_ingest/utils/pydantic_models.py,sha256=BT_j15e4rX40wQbt8LUXbqfPhA3rJn1PHTI_G_A_EHY,1720
 unstructured_ingest/utils/string_and_date_utils.py,sha256=oXOI6rxXq-8ncbk7EoJK0WCcTXWj75EzKl8pfQMID3U,2522
 unstructured_ingest/utils/table.py,sha256=WZechczgVFvlodUWFcsnCGvBNh1xRm6hr0VbJTPxKAc,3669
-unstructured_ingest-1.0.4.dist-info/METADATA,sha256=ZrV3WL4OOzjU53IKTL59o3dr5UIDRrSGewp-tFGfSF8,8719
-unstructured_ingest-1.0.4.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-unstructured_ingest-1.0.4.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
-unstructured_ingest-1.0.4.dist-info/licenses/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
-unstructured_ingest-1.0.4.dist-info/RECORD,,
+unstructured_ingest-1.0.6.dev0.dist-info/METADATA,sha256=2A8V0IxykBKTDpalgDOmkRw98MoN6whUPI0DwHkBuBc,8724
+unstructured_ingest-1.0.6.dev0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+unstructured_ingest-1.0.6.dev0.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
+unstructured_ingest-1.0.6.dev0.dist-info/licenses/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
+unstructured_ingest-1.0.6.dev0.dist-info/RECORD,,