unstructured-ingest 1.0.35__tar.gz → 1.0.37__tar.gz

{unstructured_ingest-1.0.35 → unstructured_ingest-1.0.37}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: unstructured_ingest
-Version: 1.0.35
+Version: 1.0.37
 Summary: Local ETL data pipeline to get data RAG ready
 Author-email: Unstructured Technologies <devops@unstructuredai.io>
 License-Expression: Apache-2.0

unstructured_ingest-1.0.37/unstructured_ingest/__version__.py

@@ -0,0 +1 @@
+__version__ = "1.0.37"  # pragma: no cover

{unstructured_ingest-1.0.35 → unstructured_ingest-1.0.37}/unstructured_ingest/processes/connectors/fsspec/fsspec.py

@@ -343,10 +343,9 @@ class FsspecUploader(Uploader):
             raise self.wrap_error(e=e)
 
     def get_upload_path(self, file_data: FileData) -> Path:
-        upload_path = (
-            Path(self.upload_config.path_without_protocol)
-            / file_data.source_identifiers.relative_path.lstrip("/")
-        )
+        upload_path = Path(
+            self.upload_config.path_without_protocol
+        ) / file_data.source_identifiers.relative_path.lstrip("/")
         updated_upload_path = upload_path.parent / f"{upload_path.name}.json"
         return updated_upload_path
 

unstructured_ingest-1.0.37/unstructured_ingest/processes/connectors/notion/types/blocks/synced_block.py

@@ -0,0 +1,110 @@
+# https://developers.notion.com/reference/block#synced-block
+from dataclasses import dataclass, field
+from typing import List, Optional
+
+from htmlBuilder.tags import HtmlTag
+
+from unstructured_ingest.processes.connectors.notion.interfaces import BlockBase
+
+
+@dataclass
+class OriginalSyncedBlock(BlockBase):
+    synced_from: Optional[str] = None
+    children: List[dict] = field(default_factory=list)
+
+    @staticmethod
+    def can_have_children() -> bool:
+        return True
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        """Create OriginalSyncedBlock from dictionary data.
+        
+        Original blocks contain children content.
+        """
+        if "children" not in data:
+             raise ValueError(f"OriginalSyncedBlock data missing 'children': {data}")
+        return cls(children=data["children"])
+
+    def get_html(self) -> Optional[HtmlTag]:
+        return None
+
+
+@dataclass
+class DuplicateSyncedBlock(BlockBase):
+    type: str
+    block_id: str
+
+    @staticmethod
+    def can_have_children() -> bool:
+        """Check if duplicate synced blocks can have children.
+        
+        Duplicate blocks themselves don't have children directly fetched here,
+        but they represent content that does, so Notion API might report has_children=True
+        on the parent block object. The actual children are fetched from the original block.
+        """
+        return True
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        """Create DuplicateSyncedBlock from dictionary data.
+        
+        Duplicate blocks contain a 'synced_from' reference.
+        """
+        synced_from_data = data.get("synced_from")
+        if not synced_from_data or not isinstance(synced_from_data, dict):
+            raise ValueError(f"Invalid data structure for DuplicateSyncedBlock: {data}")
+        # Ensure required keys are present in the nested dictionary
+        if "type" not in synced_from_data or "block_id" not in synced_from_data:
+            raise ValueError(
+                f"Missing 'type' or 'block_id' in synced_from data: {synced_from_data}"
+            )
+        return cls(type=synced_from_data["type"], block_id=synced_from_data["block_id"])
+
+    def get_html(self) -> Optional[HtmlTag]:
+        """Get HTML representation of the duplicate synced block.
+        
+        HTML representation might need fetching the original block's content,
+        which is outside the scope of this simple data class.
+        """
+        return None
+
+
+class SyncBlock(BlockBase):
+    @staticmethod
+    def can_have_children() -> bool:
+        """Check if synced blocks can have children.
+        
+        Synced blocks (both original and duplicate) can conceptually have children.
+        """
+        return True
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        """Create appropriate SyncedBlock subclass from dictionary data.
+        
+        Determine if it's a duplicate (has 'synced_from') or original (has 'children').
+        """
+        if data.get("synced_from") is not None:
+            # It's a duplicate block containing a reference
+            return DuplicateSyncedBlock.from_dict(data)
+        elif "children" in data:
+            # It's an original block containing children
+            return OriginalSyncedBlock.from_dict(data)
+        else:
+            # Handle cases where neither 'synced_from' nor 'children' are present.
+            # Notion API might return this for an empty original synced block.
+            # Let's treat it as an empty OriginalSyncedBlock.
+            # If this assumption is wrong, errors might occur later.
+            # Consider logging a warning here if strictness is needed.
+            return OriginalSyncedBlock(children=[])
+
+
+    def get_html(self) -> Optional[HtmlTag]:
+        """Get HTML representation of the synced block.
+        
+        The specific instance returned by from_dict (Original or Duplicate)
+        will handle its own get_html logic.
+        This method on the base SyncBlock might not be directly called.
+        """
+        return None

{unstructured_ingest-1.0.35 → unstructured_ingest-1.0.37}/unstructured_ingest/processes/connectors/sharepoint.py

@@ -29,6 +29,7 @@ from unstructured_ingest.utils.dep_check import requires_dependencies
 
 if TYPE_CHECKING:
     from office365.onedrive.driveitems.driveItem import DriveItem
+    from office365.onedrive.sites.site import Site
 
 CONNECTOR_TYPE = "sharepoint"
 LEGACY_DEFAULT_PATH = "Shared Documents"
@@ -51,6 +52,33 @@ class SharepointConnectionConfig(OnedriveConnectionConfig):
                     https://[tenant]-admin.sharepoint.com.\
                     This requires the app to be registered at a tenant level"
     )
+    library: Optional[str] = Field(
+        default=None,
+        description="Sharepoint library name. If not provided, the default \
+                    drive will be used.",
+    )
+
+    def _get_drive_item(self, client_site: Site) -> DriveItem:
+        """Helper method to get the drive item for the specified library or default drive."""
+        site_drive_item = None
+        if self.library:
+            for drive in client_site.drives.get().execute_query():
+                if drive.name == self.library:
+                    logger.info(f"Found the requested library: {self.library}")
+                    site_drive_item = drive.get().execute_query().root
+                    break
+
+        # If no specific library was found or requested, use the default drive
+        if not site_drive_item:
+            if self.library:
+                logger.warning(
+                    f"Library '{self.library}' not found in site '{self.site}'. "
+                    "Using the default drive instead."
+                )
+
+            site_drive_item = client_site.drive.get().execute_query().root
+
+        return site_drive_item
 
 
 class SharepointIndexerConfig(OnedriveIndexerConfig):
@@ -76,8 +104,8 @@ class SharepointIndexer(OnedriveIndexer):
 
         client = await asyncio.to_thread(self.connection_config.get_client)
         try:
-            site = client.sites.get_by_url(self.connection_config.site).get().execute_query()
-            site_drive_item = site.drive.get().execute_query().root
+            client_site = client.sites.get_by_url(self.connection_config.site).get().execute_query()
+            site_drive_item = self.connection_config._get_drive_item(client_site)
         except ClientRequestException:
             logger.info("Site not found")
 
@@ -118,8 +146,8 @@ class SharepointDownloader(OnedriveDownloader):
         client = self.connection_config.get_client()
 
         try:
-            site = client.sites.get_by_url(self.connection_config.site).get().execute_query()
-            site_drive_item = site.drive.get().execute_query().root
+            client_site = client.sites.get_by_url(self.connection_config.site).get().execute_query()
+            site_drive_item = self.connection_config._get_drive_item(client_site)
         except ClientRequestException:
             logger.info("Site not found")
         file = site_drive_item.get_by_path(server_relative_path).get().execute_query()

unstructured_ingest-1.0.35/unstructured_ingest/__version__.py

@@ -1 +0,0 @@
-__version__ = "1.0.35"  # pragma: no cover

unstructured_ingest-1.0.35/unstructured_ingest/processes/connectors/notion/types/blocks/synced_block.py

@@ -1,57 +0,0 @@
-# https://developers.notion.com/reference/block#synced-block
-from dataclasses import dataclass, field
-from typing import List, Optional
-
-from htmlBuilder.tags import HtmlTag
-
-from unstructured_ingest.processes.connectors.notion.interfaces import BlockBase
-
-
-@dataclass
-class OriginalSyncedBlock(BlockBase):
-    synced_from: Optional[str] = None
-    children: List[dict] = field(default_factory=list)
-
-    @staticmethod
-    def can_have_children() -> bool:
-        return True
-
-    @classmethod
-    def from_dict(cls, data: dict):
-        return cls(children=data["children"])
-
-    def get_html(self) -> Optional[HtmlTag]:
-        return None
-
-
-@dataclass
-class DuplicateSyncedBlock(BlockBase):
-    type: str
-    block_id: str
-
-    @staticmethod
-    def can_have_children() -> bool:
-        return True
-
-    @classmethod
-    def from_dict(cls, data: dict):
-        return cls(**data)
-
-    def get_html(self) -> Optional[HtmlTag]:
-        return None
-
-
-class SyncBlock(BlockBase):
-    @staticmethod
-    def can_have_children() -> bool:
-        return True
-
-    @classmethod
-    def from_dict(cls, data: dict):
-        if "synced_from" in data:
-            return OriginalSyncedBlock.from_dict(data)
-        else:
-            return DuplicateSyncedBlock.from_dict(data)
-
-    def get_html(self) -> Optional[HtmlTag]:
-        return None