streamlit-octostar-utils 0.4.2.dev1__tar.gz → 0.4.2.dev3__tar.gz

{streamlit_octostar_utils-0.4.2.dev1 → streamlit_octostar_utils-0.4.2.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: streamlit-octostar-utils
-Version: 0.4.2.dev1
+Version: 0.4.2.dev3
 Summary: 
 License: MIT
 License-File: LICENSE

{streamlit_octostar_utils-0.4.2.dev1 → streamlit_octostar_utils-0.4.2.dev3}/pyproject.toml

@@ -5,7 +5,7 @@ include = '\.pyi?$'
 
 [tool.poetry]
 name = "streamlit-octostar-utils"
-version = "0.4.2-dev.1"
+version = "0.4.2-dev.3"
 description = ""
 license = "MIT"
 authors = ["Octostar"]

{streamlit_octostar_utils-0.4.2.dev1 → streamlit_octostar_utils-0.4.2.dev3}/streamlit_octostar_utils/api_crafter/contents.py

@@ -11,6 +11,7 @@ Provides a FileIO-like interface for handling entity contents with support for:
 Storage Backends:
 - MemoryContents: In-memory buffer (like BytesIO)
 - WorkspaceAttachmentContents: Octostar workspace attachments with HTTP Range support
+- TemporaryAttachmentContents: Octostar temporary blob storage (user temp bucket)
 """
 
 from abc import ABC, abstractmethod
@@ -29,6 +30,7 @@ class ContentsLocation(Enum):
     """Enumeration of supported content storage locations."""
     MEMORY = "memory"
     WORKSPACE_ATTACHMENT = "workspace_attachment"
+    TEMPORARY_ATTACHMENT = "temporary_attachment"
 
 
 class Contents(ABC):
@@ -42,12 +44,10 @@ class Contents(ABC):
     
     def __init__(
         self,
-        mode: str = "rb",
         entity_type: Optional[str] = None,
         filetype: Optional[str] = None,
         **kwargs
     ):
-        self._mode = mode
         self._entity_type = entity_type
         self._filetype = filetype
         self._closed = False
@@ -105,15 +105,12 @@ class Contents(ABC):
         pass
     
     def readable(self) -> bool:
-        """Check if stream is readable."""
-        return 'r' in self._mode or '+' in self._mode
+        return True
     
     def writable(self) -> bool:
-        """Check if stream is writable."""
-        return 'w' in self._mode or 'a' in self._mode or '+' in self._mode
+        return True
     
     def seekable(self) -> bool:
-        """Check if stream is seekable."""
         return True
     
     @abstractmethod
@@ -126,16 +123,16 @@ class Contents(ABC):
         """Close the stream and release resources."""
         self._closed = True
     
+    @abstractmethod
+    def delete(self):
+        """Delete the contents from their storage backend."""
+        pass
+    
     @property
     def closed(self) -> bool:
         """Check if stream is closed."""
         return self._closed
     
-    @property
-    def mode(self) -> str:
-        """Get the file mode."""
-        return self._mode
-    
     @abstractmethod
     def truncate(self, size: Optional[int] = None) -> int:
         """
@@ -286,6 +283,8 @@ class Contents(ABC):
                 return MemoryContents._from_locator(locator)
             case ContentsLocation.WORKSPACE_ATTACHMENT.value:
                 return WorkspaceAttachmentContents._from_locator(locator, client)
+            case ContentsLocation.TEMPORARY_ATTACHMENT.value:
+                return TemporaryAttachmentContents._from_locator(locator, client)
             case _:
                 raise ValueError(f"Unknown contents location type: {location}")
     
@@ -336,24 +335,19 @@ class MemoryContents(Contents):
     
     def __init__(
         self,
-        mode: str = "r+b",
         entity_type: Optional[str] = None,
         filetype: Optional[str] = None,
         *,
         initial_data: Optional[bytes] = None,
         **kwargs
     ):
-        super().__init__(mode, entity_type, filetype, **kwargs)
+        super().__init__(entity_type, filetype, **kwargs)
         self._buffer = BytesIO(initial_data or b"")
     
     def read(self, size: int = -1) -> bytes:
-        if not self.readable():
-            raise IOError("Contents not readable")
         return self._buffer.read(size)
     
     def write(self, b: bytes) -> int:
-        if not self.writable():
-            raise IOError("Contents not writable")
         return self._buffer.write(b)
     
     def seek(self, offset: int, whence: int = SEEK_SET) -> int:
@@ -370,6 +364,10 @@ class MemoryContents(Contents):
             self._buffer.close()
             super().close()
     
+    def delete(self):
+        """Clear the in-memory buffer."""
+        self._buffer = BytesIO(b"")
+    
     def truncate(self, size: Optional[int] = None) -> int:
         return self._buffer.truncate(size)
     
@@ -435,7 +433,6 @@ class WorkspaceAttachmentContents(Contents):
     
     def __init__(
         self,
-        mode: str = "rb",
         entity_type: Optional[str] = None,
         filetype: Optional[str] = None,
         *,
@@ -446,7 +443,7 @@ class WorkspaceAttachmentContents(Contents):
         chunk_size: int = DEFAULT_CHUNK_SIZE,
         **kwargs
     ):
-        super().__init__(mode, entity_type, filetype, **kwargs)
+        super().__init__(entity_type, filetype, **kwargs)
         self._workspace_id = workspace_id
         self._entity_id = entity_id
         self._client = client
@@ -575,11 +572,7 @@ class WorkspaceAttachmentContents(Contents):
         self._position = 0
     
     def read(self, size: int = -1) -> bytes:
-        if not self.readable():
-            raise IOError("Contents not readable")
-        
-        # If writable or already fully loaded, use buffer
-        if self.writable() or self._fully_loaded:
+        if self._fully_loaded:
             if not self._buffer:
                 self._load_full()
             return self._buffer.read(size)
@@ -613,9 +606,6 @@ class WorkspaceAttachmentContents(Contents):
         return data
     
     def write(self, b: bytes) -> int:
-        if not self.writable():
-            raise IOError("Contents not writable")
-        
         if not self._buffer:
             self._load_full()
         
@@ -684,10 +674,8 @@ class WorkspaceAttachmentContents(Contents):
     
     def close(self):
         if not self._closed:
-            # Flush any pending writes before closing
             if self._modified:
                 self.flush()
-            
             if self._buffer:
                 self._buffer.close()
             if self._http_client:
@@ -695,6 +683,15 @@ class WorkspaceAttachmentContents(Contents):
                 self._http_client = None
             super().close()
     
+    def delete(self):
+        """Delete the entity from the workspace using delete_entity()."""
+        from octostar.utils.workspace import delete_entity
+        
+        delete_entity.sync(
+            os_entity_uid=self._entity_id,
+            client=self._client
+        )
+    
     def truncate(self, size: Optional[int] = None) -> int:
         if not self._buffer:
             self._load_full()
@@ -758,3 +755,160 @@ class WorkspaceAttachmentContents(Contents):
             entity_id=entity_id,
             client=client
         )
+
+
+class TemporaryAttachmentContents(Contents):
+    """
+    Contents implementation for Octostar temporary blob storage.
+    
+    Uses octostar-api utilities (read_temporary_blob, write_temporary_blob,
+    delete_temporary_blob) to store files in the user's temporary S3 bucket.
+    
+    Temporary blobs are keyed by filename (not workspace/entity), and are not
+    associated with any workspace entity. Use WorkspaceAttachmentContents for that.
+    """
+    
+    def __init__(
+        self,
+        entity_type: Optional[str] = None,
+        filetype: Optional[str] = None,
+        *,
+        filename: str,
+        client,
+        initial_data: Optional[bytes] = None,
+        **kwargs
+    ):
+        super().__init__(entity_type, filetype, **kwargs)
+        self._filename = filename
+        self._client = client
+        
+        self._buffer: Optional[BytesIO] = None
+        self._fully_loaded = False
+        self._modified = False
+        
+        if initial_data is not None:
+            self._buffer = BytesIO(initial_data)
+            self._fully_loaded = True
+    
+    def _load_full(self):
+        """Load the entire blob into memory using read_temporary_blob()."""
+        if self._fully_loaded:
+            return
+        
+        from octostar.utils.workspace import read_temporary_blob
+        
+        data = read_temporary_blob.sync(
+            filename=self._filename,
+            decode=False,
+            client=self._client
+        )
+        self._buffer = BytesIO(data or b"")
+        self._fully_loaded = True
+    
+    def read(self, size: int = -1) -> bytes:
+        if not self._buffer:
+            self._load_full()
+        return self._buffer.read(size)
+    
+    def write(self, b: bytes) -> int:
+        if not self._buffer:
+            self._load_full()
+        n = self._buffer.write(b)
+        self._modified = True
+        return n
+    
+    def seek(self, offset: int, whence: int = SEEK_SET) -> int:
+        if not self._buffer:
+            self._load_full()
+        return self._buffer.seek(offset, whence)
+    
+    def tell(self) -> int:
+        if not self._buffer:
+            self._load_full()
+        return self._buffer.tell()
+    
+    def flush(self):
+        """Flush the internal buffer and write to temp bucket if modified."""
+        if self._buffer:
+            self._buffer.flush()
+        
+        if not self._modified or not self._buffer:
+            return
+        
+        from octostar.utils.workspace import write_temporary_blob
+        
+        current_pos = self._buffer.tell()
+        self._buffer.seek(0, SEEK_SET)
+        data = self._buffer.read()
+        self._buffer.seek(current_pos, SEEK_SET)
+        
+        write_temporary_blob.sync(
+            filename=self._filename,
+            file=data,
+            client=self._client
+        )
+        self._modified = False
+    
+    def close(self):
+        if not self._closed:
+            if self._modified:
+                self.flush()
+            if self._buffer:
+                self._buffer.close()
+            super().close()
+    
+    def delete(self):
+        """Delete the blob from the temporary bucket."""
+        from octostar.utils.workspace import delete_temporary_blob
+        
+        delete_temporary_blob.sync(
+            filename=self._filename,
+            client=self._client
+        )
+    
+    def truncate(self, size: Optional[int] = None) -> int:
+        if not self._buffer:
+            self._load_full()
+        self._modified = True
+        return self._buffer.truncate(size)
+    
+    def getvalue(self) -> bytes:
+        if not self._buffer or not self._fully_loaded:
+            self._load_full()
+        return self._buffer.getvalue()
+    
+    def to_locator(self) -> Dict[str, Any]:
+        """
+        Serialize to locator with filename.
+        
+        Returns:
+            {"location": "temporary_attachment", "filename": "..."}
+        """
+        locator = {
+            "location": ContentsLocation.TEMPORARY_ATTACHMENT.value,
+            "filename": self._filename
+        }
+        if self._entity_type:
+            locator["entity_type"] = self._entity_type
+        if self._filetype:
+            locator["filetype"] = self._filetype
+        return locator
+    
+    @staticmethod
+    def _from_locator(locator: Dict[str, Any], client=None) -> 'TemporaryAttachmentContents':
+        """
+        Create TemporaryAttachmentContents from a locator dictionary.
+        
+        Args:
+            locator: Locator dictionary with filename
+            client: Octostar client
+            
+        Returns:
+            New TemporaryAttachmentContents instance
+        """
+        return TemporaryAttachmentContents(
+            entity_type=locator.get("entity_type"),
+            filetype=locator.get("filetype"),
+            filename=locator["filename"],
+            client=client
+        )

{streamlit_octostar_utils-0.4.2.dev1 → streamlit_octostar_utils-0.4.2.dev3}/streamlit_octostar_utils/api_crafter/nifi.py

@@ -5,18 +5,16 @@ from contextlib import contextmanager
 import json
 import jwt
 from typing import List, Union, Optional, Callable
-import base64
 from pydantic import BaseModel, ConfigDict, Field
 import itertools
-import asyncio
 from enum import Enum
-from typing import Dict, Any
+from typing import Dict, Any, Literal
 from datetime import datetime, timezone, timedelta
 from fastapi import Request
 from fastapi.responses import JSONResponse
 from starlette.exceptions import HTTPException as StarletteHTTPException
 
-from octostar.utils.workspace import read_file, upsert_entities, write_file
+from octostar.utils.workspace import upsert_entities, write_file
 from octostar.utils.ontology import fetch_ontology_data
 from octostar.utils.workspace.permissions import get_permissions, PermissionLevel
 from octostar.client import make_client
@@ -26,26 +24,23 @@ from ..core.dict import recursive_update_dict, travel_dict, jsondict_hash
 from ..core.timestamp import now, string_to_datetime
 from .fastapi import DefaultErrorRoute, Route
 from ..ontology.inheritance import is_child_concept as is_child_concept_fn, get_label_keys
-from .contents import Contents, MemoryContents, WorkspaceAttachmentContents, ContentsLocation
+from .contents import Contents, WorkspaceAttachmentContents, TemporaryAttachmentContents, ContentsLocation
+
+
+class RelationshipName(BaseModel):
+    name: str
+    type: Literal["mtm", "otm"]
+
 
 RELATIONSHIP_ENTITY_NAME = "os_relationship"
 LOCAL_RELATIONSHIP_ENTITY_NAME = "os_workspace_relationship"
 FILE_ENTITY_NAME = "os_file"
-GENERIC_RELATIONSHIP_NAME = "related_to"
-FILE_RELATIONSHIP_NAME = "generator_of"
-TAG_RELATIONSHIP_NAME = "has_tag"
 TAG_ENTITY_NAME = "os_tag"
-
-
-def safe_async_run(coro):
-    try:
-        loop = asyncio.get_event_loop()
-        if loop.is_running():
-            return asyncio.ensure_future(coro)
-        else:
-            return asyncio.run(coro)
-    except RuntimeError:
-        return asyncio.run(coro)
+FRAGMENT_ENTITY_NAME = "os_fragment"
+GENERIC_RELATIONSHIP = RelationshipName(name="related_to", type="mtm")
+FILE_RELATIONSHIP = RelationshipName(name="generator_of", type="mtm")
+TAG_RELATIONSHIP = RelationshipName(name="has_tag", type="mtm")
+FRAGMENT_RELATIONSHIP = RelationshipName(name="is_fragment_of", type="otm")
 
 
 class NifiProxyEntityModel(BaseModel):
@@ -397,10 +392,6 @@ class NifiContextManager(object):
 
     @contextmanager
     def reindex_lock(self, entities: List[Union[dict, "NifiEntity"]], timeout: int = 7200):
-        """Lock entities to prevent reindexing during processing.
-        
-        Sets do_not_reindex_at to a future time on entry, resets on exit.
-        """
         if not entities:
             yield
             return
@@ -453,11 +444,14 @@ class NifiContextManager(object):
         fetch_concept_relationships = {}
         # FIND FILES TO WRITE
         for entity in entities:
-            if entity.is_child_concept("os_file"):
+            if entity.is_child_concept("os_attachable"):
                 has_write_flag = entity.sync_params.get(NifiContextManager.SyncFlag.WRITE_CONTENTS)
-                is_temp_with_pointer = entity.request.get("is_temporary") and entity.contents_pointer
-                if has_write_flag or is_temp_with_pointer:
-                    if entity.contents:  # Contents instance check
+                is_temporary = entity.request.get("is_temporary")
+                if is_temporary:
+                    if entity._contents and not isinstance(entity._contents, WorkspaceAttachmentContents):
+                        files_to_write.append(entity)
+                elif has_write_flag:
+                    if entity.contents:
                         files_to_write.append(entity)
         # FIND ENTITIES TO UPSERT
         self._find_entities_to_upsert(entities, entities_to_upsert, reserved_fields)
@@ -487,8 +481,7 @@ class NifiContextManager(object):
                 for file in files_to_write:
                     if not file.contents:
                         continue
-                    
-                    # Pass Contents instance directly — write_file uses duck typing
+                    old_contents = file._contents
                     new_file_record = write_file.sync(
                         file.write_os_workspace,
                         "./" + file.record["os_item_name"],
@@ -517,6 +510,11 @@ class NifiContextManager(object):
                         "entity_type": file.record["os_concept"],
                         "filetype": file.record["os_item_content_type"]
                     }
+                    if isinstance(old_contents, TemporaryAttachmentContents):
+                        try:
+                            old_contents.delete()
+                        except Exception:
+                            pass
             # UPSERT ENTITIES
             if entities_to_upsert:
                 new_entities = upsert_entities.sync(
@@ -572,8 +570,7 @@ class NifiContextManager(object):
                             target["os_workspace"],
                             target["os_concept"],
                             target,
-                            os_relationship_name=rel["os_relationship_name"],
-                            os_relationship_type=rel_type,
+                            relationship=RelationshipName(name=rel["os_relationship_name"], type=rel_type),
                         )
                         child_entity.record = {**child_entity.record, **target}
                         child_entity.record["entity_id"] = child_entity.record[
@@ -967,7 +964,7 @@ class NifiEntity(object):
     ):
         return self._add_entity(
             os_relationship_workspace,
-            (LOCAL_RELATIONSHIP_ENTITY_NAME if os_relationship_workspace else RELATIONSHIP_ENTITY_NAME),
+            LOCAL_RELATIONSHIP_ENTITY_NAME,
             {
                 **relationship_fields,
                 "os_entity_uid_from": os_entity_uid_from,
@@ -1002,33 +999,32 @@ class NifiEntity(object):
         os_workspace,
         entity_type,
         fields,
-        os_relationship_name=GENERIC_RELATIONSHIP_NAME,
-        os_relationship_type="mtm",
+        relationship: RelationshipName = GENERIC_RELATIONSHIP,
         os_entity_uid=None,
         os_relationship_uid=None,
     ):
         child_entity = self._add_entity(os_workspace, entity_type, fields, os_entity_uid)
-        if os_relationship_type == "mtm":
+        if relationship.type == "mtm":
             child_rel = self.add_mtm_relationship(
                 self.record["os_entity_uid"],
                 self.record["os_concept"],
                 child_entity.record["os_entity_uid"],
                 child_entity.record["os_concept"],
-                os_relationship_name,
+                relationship.name,
                 os_workspace,
                 {},
                 os_relationship_uid,
             )
-        elif os_relationship_type == "otm":
+        elif relationship.type == "otm":
             child_rel = self.add_otm_relationship(
                 self.record["os_entity_uid"],
                 self.record["os_concept"],
                 child_entity.record["os_entity_uid"],
                 child_entity.record["os_concept"],
-                os_relationship_name,
+                relationship.name,
             )
         else:
-            raise ValueError(f"os_relationship_type is invalid! {os_relationship_type}")
+            raise ValueError(f"relationship.type is invalid! {relationship.type}")
         return child_entity, child_rel
 
     def add_child_file(
@@ -1039,14 +1035,14 @@ class NifiEntity(object):
         filetype,
         file: Union[Contents, bytes],
         fields={},
-        os_relationship_name=FILE_RELATIONSHIP_NAME,
-        os_relationship_type="mtm",
+        relationship: RelationshipName = FILE_RELATIONSHIP,
         os_entity_uid=None,
         os_relationship_uid=None,
+        os_entity_type=FILE_ENTITY_NAME,
     ):
         child_entity, child_rel = self.add_child_entity(
             os_workspace,
-            FILE_ENTITY_NAME,
+            os_entity_type,
             {
                 **fields,
                 "os_item_name": filename,
@@ -1054,27 +1050,58 @@ class NifiEntity(object):
                 "os_has_attachment": True,
                 "os_parent_folder": os_parent_folder,
             },
-            os_relationship_name,
-            os_relationship_type,
+            relationship,
             os_entity_uid,
             os_relationship_uid,
         )
         if isinstance(file, Contents):
             child_entity._contents = file
         else:
-            child_entity._contents = MemoryContents(
-                entity_type=FILE_ENTITY_NAME, filetype=filetype, initial_data=file
+            temp_filename = f"tmp_{child_entity.record['os_entity_uid']}"
+            temp_contents = TemporaryAttachmentContents(
+                entity_type=os_entity_type,
+                filetype=filetype,
+                filename=temp_filename,
+                client=self.context.client,
+                initial_data=file,
             )
+            temp_contents.flush()
+            child_entity._contents = temp_contents
         child_entity.request["contents_pointer"] = child_entity._contents.to_locator()
         return child_entity, child_rel
 
+    def add_child_fragment(
+        self,
+        os_workspace,
+        os_parent_folder,
+        filename,
+        filetype,
+        file: Union[Contents, bytes],
+        fields={},
+        relationship: RelationshipName = FRAGMENT_RELATIONSHIP,
+        os_entity_uid=None,
+        os_relationship_uid=None,
+        os_entity_type=FRAGMENT_ENTITY_NAME,
+    ):
+        return self.add_child_file(
+            os_workspace,
+            os_parent_folder,
+            filename,
+            filetype,
+            file,
+            fields,
+            relationship,
+            os_entity_uid,
+            os_relationship_uid,
+            os_entity_type,
+        )
+
     def add_tag(self, os_workspace, name, group, order, color, fields={}):
         return self.add_child_entity(
             os_workspace,
-            "os_tag",
-            {**fields,"name": name, "group": group, "order": order, "color": color},
-            TAG_RELATIONSHIP_NAME,
-            "mtm",
+            TAG_ENTITY_NAME,
+            {**fields, "name": name, "group": group, "order": order, "color": color},
+            TAG_RELATIONSHIP,
         )
 
     def add_annotations(