unstructured-ingest 1.0.4__tar.gz → 1.0.6.dev0__tar.gz

{unstructured_ingest-1.0.4 → unstructured_ingest-1.0.6.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: unstructured_ingest
-Version: 1.0.4
+Version: 1.0.6.dev0
 Summary: Local ETL data pipeline to get data RAG ready
 Author-email: Unstructured Technologies <devops@unstructuredai.io>
 License-Expression: Apache-2.0

unstructured_ingest-1.0.6.dev0/unstructured_ingest/__version__.py

@@ -0,0 +1 @@
+__version__ = "1.0.6-dev0"  # pragma: no cover

{unstructured_ingest-1.0.4 → unstructured_ingest-1.0.6.dev0}/unstructured_ingest/data_types/file_data.py

@@ -1,6 +1,6 @@
 import json
 from pathlib import Path
-from typing import Any, Optional, Union
+from typing import Any, Optional
 from uuid import NAMESPACE_DNS, uuid5
 
 from pydantic import BaseModel, Field, ValidationError, field_validator, model_validator
@@ -29,7 +29,7 @@ class FileDataSourceMetadata(BaseModel):
     date_created: Optional[str] = None
     date_modified: Optional[str] = None
     date_processed: Optional[str] = None
-    permissions_data: Union[list[dict[str, Any]], dict[str, Any], None] = None
+    permissions_data: Optional[list[dict[str, Any]]] = None
     filesize_bytes: Optional[int] = None
 
 

{unstructured_ingest-1.0.4 → unstructured_ingest-1.0.6.dev0}/unstructured_ingest/processes/connectors/confluence.py

@@ -352,6 +352,7 @@ class ConfluenceDownloader(Downloader):
     def _get_permissions_for_space(self, space_id: int) -> Optional[List[dict]]:
         if space_id in self._permissions_cache:
             self._permissions_cache.move_to_end(space_id)  # mark recent use
+            logger.debug(f"Retrieved cached permissions for space {space_id}")
             return self._permissions_cache[space_id]
         else:
             with self.connection_config.get_client() as client:
@@ -371,22 +372,27 @@ class ConfluenceDownloader(Downloader):
                         self._permissions_cache.popitem(last=False)  # LRU/FIFO eviction
                     self._permissions_cache[space_id] = space_permissions
 
+                    logger.debug(f"Retrieved permissions for space {space_id}")
                     return space_permissions
                 except Exception as e:
                     logger.debug(f"Could not retrieve permissions for space {space_id}: {e}")
                     return None
 
-    def _parse_permissions_for_doc(self, doc_id: str, space_permissions: list) -> Optional[dict]:
+    def _parse_permissions_for_doc(
+        self, doc_id: str, space_permissions: list
+    ) -> Optional[list[dict]]:
         with self.connection_config.get_client() as client:
             try:
                 doc_permissions = client.get_all_restrictions_for_content(content_id=doc_id)
                 parsed_permissions_dict = self.parse_permissions(doc_permissions, space_permissions)
+                parsed_permissions_dict = [{k: v} for k, v in parsed_permissions_dict.items()]
 
             except Exception as e:
                 # skip writing any permission metadata
                 logger.debug(f"Could not retrieve permissions for doc {doc_id}: {e}")
                 return None
 
+        logger.debug(f"normalized permissions generated: {parsed_permissions_dict}")
         return parsed_permissions_dict
 
     def run(self, file_data: FileData, **kwargs) -> download_responses:

{unstructured_ingest-1.0.4 → unstructured_ingest-1.0.6.dev0}/unstructured_ingest/processes/connectors/google_drive.py

@@ -54,7 +54,8 @@ class GoogleDriveAccessConfig(AccessConfig):
         default=None, description="Credentials values to use for authentication"
     )
     service_account_key_path: Optional[Path] = Field(
-        default=None, description="File path to credentials values to use for authentication"
+        default=None,
+        description="File path to credentials values to use for authentication",
     )
 
     def model_post_init(self, __context: Any) -> None:
@@ -111,10 +112,9 @@ class GoogleDriveIndexerConfig(IndexerConfig):
     extensions: Optional[list[str]] = None
     recursive: bool = False
 
-    def __post_init__(self):
-        # Strip leading period of extension
+    def model_post_init(self, __context: Any) -> None:
         if self.extensions is not None:
-            self.extensions = [e[1:] if e.startswith(".") else e for e in self.extensions]
+            self.extensions = [e.lstrip(".") for e in self.extensions]
 
 
 @dataclass
@@ -275,7 +275,8 @@ class GoogleDriveIndexer(Indexer):
 
         except Exception as e:
             logger.error(
-                "Failed to validate Google Drive connection during precheck", exc_info=True
+                "Failed to validate Google Drive connection during precheck",
+                exc_info=True,
             )
             raise SourceConnectionError(f"Precheck failed: {e}")
 
@@ -284,17 +285,17 @@ class GoogleDriveIndexer(Indexer):
         return record.get("mimeType") == "application/vnd.google-apps.folder"
 
     @staticmethod
-    def map_file_data(f: dict) -> FileData:
-        file_id = f["id"]
-        filename = f.pop("name")
-        url = f.pop("webContentLink", None)
-        version = f.pop("version", None)
-        permissions = f.pop("permissions", None)
-        date_created_str = f.pop("createdTime", None)
+    def map_file_data(root_info: dict) -> FileData:
+        file_id = root_info["id"]
+        filename = root_info.pop("name")
+        url = root_info.pop("webContentLink", None)
+        version = root_info.pop("version", None)
+        permissions = root_info.pop("permissions", None)
+        date_created_str = root_info.pop("createdTime", None)
         date_created_dt = parser.parse(date_created_str) if date_created_str else None
-        date_modified_str = f.pop("modifiedTime", None)
-        parent_path = f.pop("parent_path", None)
-        parent_root_path = f.pop("parent_root_path", None)
+        date_modified_str = root_info.pop("modifiedTime", None)
+        parent_path = root_info.pop("parent_path", None)
+        parent_root_path = root_info.pop("parent_root_path", None)
         date_modified_dt = parser.parse(date_modified_str) if date_modified_str else None
         if (
             parent_path
@@ -323,7 +324,7 @@ class GoogleDriveIndexer(Indexer):
                     "file_id": file_id,
                 },
             ),
-            additional_metadata=f,
+            additional_metadata=root_info,
         )
 
     def get_paginated_results(
@@ -404,13 +405,14 @@ class GoogleDriveIndexer(Indexer):
             data = []
             for f in file_contents:
                 f["permissions"] = self.extract_permissions(f.get("permissions"))
-                data.append(self.map_file_data(f=f))
+                data.append(self.map_file_data(root_info=f))
         for d in data:
             d.metadata.record_locator["drive_id"]: object_id
         return data
 
-    def extract_permissions(self, permissions: list[dict]) -> dict:
+    def extract_permissions(self, permissions: Optional[list[dict]]) -> list[dict]:
         if not permissions:
+            logger.debug("no permissions found")
             return {}
 
         # https://developers.google.com/workspace/drive/api/guides/ref-roles
@@ -442,7 +444,8 @@ class GoogleDriveIndexer(Indexer):
             for key in role_dict:
                 role_dict[key] = sorted(role_dict[key])
 
-        return normalized_permissions
+        logger.debug(f"normalized permissions generated: {normalized_permissions}")
+        return [{k: v} for k, v in normalized_permissions.items()]
 
     def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
         with self.connection_config.get_client() as client:

unstructured_ingest-1.0.4/unstructured_ingest/__version__.py

@@ -1 +0,0 @@
-__version__ = "1.0.4"  # pragma: no cover