unstructured-ingest 1.0.5__tar.gz → 1.0.6.dev0__tar.gz

{unstructured_ingest-1.0.5 → unstructured_ingest-1.0.6.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: unstructured_ingest
-Version: 1.0.5
+Version: 1.0.6.dev0
 Summary: Local ETL data pipeline to get data RAG ready
 Author-email: Unstructured Technologies <devops@unstructuredai.io>
 License-Expression: Apache-2.0

unstructured_ingest-1.0.6.dev0/unstructured_ingest/__version__.py

@@ -0,0 +1 @@
+__version__ = "1.0.6-dev0"  # pragma: no cover

{unstructured_ingest-1.0.5 → unstructured_ingest-1.0.6.dev0}/unstructured_ingest/data_types/file_data.py

@@ -1,6 +1,6 @@
 import json
 from pathlib import Path
-from typing import Any, Optional, Union
+from typing import Any, Optional
 from uuid import NAMESPACE_DNS, uuid5
 
 from pydantic import BaseModel, Field, ValidationError, field_validator, model_validator
@@ -29,31 +29,9 @@ class FileDataSourceMetadata(BaseModel):
     date_created: Optional[str] = None
     date_modified: Optional[str] = None
     date_processed: Optional[str] = None
-    permissions_data: Union[list[dict[str, Any]], dict[str, Any], None] = None
+    permissions_data: Optional[list[dict[str, Any]]] = None
     filesize_bytes: Optional[int] = None
 
-    @field_validator("permissions_data", mode="before")
-    @classmethod
-    def coerce_permissions_data(cls, v: Any) -> Any:
-        if isinstance(v, dict):
-            # Temporarily convert dict to list for validation
-            return [v]
-        return v
-
-    @field_validator("permissions_data", mode="after")
-    @classmethod
-    def restore_dict_permissions_data(
-        cls, v: Optional[list[dict[str, Any]]]
-    ) -> Union[list[dict[str, Any]], dict[str, Any], None]:
-        if (
-            isinstance(v, list)
-            and len(v) == 1
-            and isinstance(v[0], dict)
-            and any(isinstance(val, dict) for val in v[0].values())
-        ):
-            return v[0]
-        return v
-
 
 class FileData(BaseModel):
     identifier: str

{unstructured_ingest-1.0.5 → unstructured_ingest-1.0.6.dev0}/unstructured_ingest/processes/connectors/confluence.py

@@ -378,11 +378,14 @@ class ConfluenceDownloader(Downloader):
                     logger.debug(f"Could not retrieve permissions for space {space_id}: {e}")
                     return None
 
-    def _parse_permissions_for_doc(self, doc_id: str, space_permissions: list) -> Optional[dict]:
+    def _parse_permissions_for_doc(
+        self, doc_id: str, space_permissions: list
+    ) -> Optional[list[dict]]:
         with self.connection_config.get_client() as client:
             try:
                 doc_permissions = client.get_all_restrictions_for_content(content_id=doc_id)
                 parsed_permissions_dict = self.parse_permissions(doc_permissions, space_permissions)
+                parsed_permissions_dict = [{k: v} for k, v in parsed_permissions_dict.items()]
 
             except Exception as e:
                 # skip writing any permission metadata

{unstructured_ingest-1.0.5 → unstructured_ingest-1.0.6.dev0}/unstructured_ingest/processes/connectors/google_drive.py

@@ -54,7 +54,8 @@ class GoogleDriveAccessConfig(AccessConfig):
         default=None, description="Credentials values to use for authentication"
     )
     service_account_key_path: Optional[Path] = Field(
-        default=None, description="File path to credentials values to use for authentication"
+        default=None,
+        description="File path to credentials values to use for authentication",
     )
 
     def model_post_init(self, __context: Any) -> None:
@@ -111,10 +112,9 @@ class GoogleDriveIndexerConfig(IndexerConfig):
     extensions: Optional[list[str]] = None
     recursive: bool = False
 
-    def __post_init__(self):
-        # Strip leading period of extension
+    def model_post_init(self, __context: Any) -> None:
         if self.extensions is not None:
-            self.extensions = [e[1:] if e.startswith(".") else e for e in self.extensions]
+            self.extensions = [e.lstrip(".") for e in self.extensions]
 
 
 @dataclass
@@ -275,7 +275,8 @@ class GoogleDriveIndexer(Indexer):
 
         except Exception as e:
             logger.error(
-                "Failed to validate Google Drive connection during precheck", exc_info=True
+                "Failed to validate Google Drive connection during precheck",
+                exc_info=True,
             )
             raise SourceConnectionError(f"Precheck failed: {e}")
 
@@ -284,17 +285,17 @@ class GoogleDriveIndexer(Indexer):
         return record.get("mimeType") == "application/vnd.google-apps.folder"
 
     @staticmethod
-    def map_file_data(f: dict) -> FileData:
-        file_id = f["id"]
-        filename = f.pop("name")
-        url = f.pop("webContentLink", None)
-        version = f.pop("version", None)
-        permissions = f.pop("permissions", None)
-        date_created_str = f.pop("createdTime", None)
+    def map_file_data(root_info: dict) -> FileData:
+        file_id = root_info["id"]
+        filename = root_info.pop("name")
+        url = root_info.pop("webContentLink", None)
+        version = root_info.pop("version", None)
+        permissions = root_info.pop("permissions", None)
+        date_created_str = root_info.pop("createdTime", None)
         date_created_dt = parser.parse(date_created_str) if date_created_str else None
-        date_modified_str = f.pop("modifiedTime", None)
-        parent_path = f.pop("parent_path", None)
-        parent_root_path = f.pop("parent_root_path", None)
+        date_modified_str = root_info.pop("modifiedTime", None)
+        parent_path = root_info.pop("parent_path", None)
+        parent_root_path = root_info.pop("parent_root_path", None)
         date_modified_dt = parser.parse(date_modified_str) if date_modified_str else None
         if (
             parent_path
@@ -323,7 +324,7 @@ class GoogleDriveIndexer(Indexer):
                     "file_id": file_id,
                 },
             ),
-            additional_metadata=f,
+            additional_metadata=root_info,
         )
 
     def get_paginated_results(
@@ -404,12 +405,12 @@ class GoogleDriveIndexer(Indexer):
             data = []
             for f in file_contents:
                 f["permissions"] = self.extract_permissions(f.get("permissions"))
-                data.append(self.map_file_data(f=f))
+                data.append(self.map_file_data(root_info=f))
         for d in data:
             d.metadata.record_locator["drive_id"]: object_id
         return data
 
-    def extract_permissions(self, permissions: Optional[list[dict]]) -> dict:
+    def extract_permissions(self, permissions: Optional[list[dict]]) -> list[dict]:
         if not permissions:
             logger.debug("no permissions found")
             return {}
@@ -444,7 +445,7 @@ class GoogleDriveIndexer(Indexer):
                 role_dict[key] = sorted(role_dict[key])
 
         logger.debug(f"normalized permissions generated: {normalized_permissions}")
-        return normalized_permissions
+        return [{k: v} for k, v in normalized_permissions.items()]
 
     def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
         with self.connection_config.get_client() as client:

unstructured_ingest-1.0.5/unstructured_ingest/__version__.py

@@ -1 +0,0 @@
-__version__ = "1.0.5"  # pragma: no cover