streamlit-octostar-utils 0.4.2.dev9__tar.gz → 0.4.2.dev10__tar.gz

{streamlit_octostar_utils-0.4.2.dev9 → streamlit_octostar_utils-0.4.2.dev10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: streamlit-octostar-utils
-Version: 0.4.2.dev9
+Version: 0.4.2.dev10
 Summary: 
 License: MIT
 License-File: LICENSE

{streamlit_octostar_utils-0.4.2.dev9 → streamlit_octostar_utils-0.4.2.dev10}/pyproject.toml

@@ -5,7 +5,7 @@ include = '\.pyi?$'
 
 [tool.poetry]
 name = "streamlit-octostar-utils"
-version = "0.4.2-dev.9"
+version = "0.4.2-dev.10"
 description = ""
 license = "MIT"
 authors = ["Octostar"]

{streamlit_octostar_utils-0.4.2.dev9 → streamlit_octostar_utils-0.4.2.dev10}/streamlit_octostar_utils/api_crafter/contents.py

@@ -15,13 +15,12 @@ Storage Backends:
 """
 
 from abc import ABC, abstractmethod
-from typing import Optional, Dict, Any, Union, BinaryIO, List
+from typing import Optional, Dict, Any, List
 from enum import Enum
 import base64
 import httpx
 import logging
 from io import BytesIO, SEEK_SET, SEEK_CUR, SEEK_END
-from copy import deepcopy
 
 _logger = logging.getLogger(__name__)
 
@@ -415,107 +414,107 @@ class MemoryContents(Contents):
         )
 
 
-class WorkspaceAttachmentContents(Contents):
+class _PresignedUrlRemoteContents(Contents):
     """
-    Contents implementation for Octostar workspace attachments.
-    
-    Uses octostar-api utilities (read_attachment, get_attachment_url) to:
-    - Lazy load from workspace storage via read_attachment()
-    - Stream efficiently with HTTP Range requests (no need to download entire file)
-    - Support true seeking with configurable chunk sizes
-    - Buffer modifications in memory (write back handled at NiFi entity level)
-    
-    Presigned URL Handling:
-    - URLs are obtained via get_attachment_url() and cached
-    - On 403 (Forbidden) responses, URLs are refreshed and requests retried
-    - Follows the same retry pattern as octostar-api read_file utilities
+    Abstract base for remote storage backends accessed via presigned URLs.
+    
+    Provides all FileIO operations using presigned URLs:
+    - Reads via HTTP GET with Range headers (no need to download entire file)
+    - Writes via presigned POST (no entity upsert, only raw bytes to S3)
+    - Automatic presigned URL refresh on 403 (expired)
+    - Lazy full-load into memory buffer only when writes require it
+    - Cache invalidation on flush (next read re-fetches from remote)
+    
+    Subclasses only need to implement:
+    - _refresh_download_url(): get a fresh presigned download URL
+    - _get_upload_info(): get presigned upload URL + form fields
+    - delete(): delete from the storage backend
+    - to_locator() / _from_locator(): serialization
     """
     
-    DEFAULT_CHUNK_SIZE = 8192
     DEFAULT_URL_TIMEOUT = 120
+    MAX_RETRIES = 3
     
     def __init__(
         self,
         entity_type: Optional[str] = None,
         filetype: Optional[str] = None,
-        *,
-        workspace_id: str,
-        entity_id: str,
-        client,
-        initial_data: Optional[bytes] = None,
-        chunk_size: int = DEFAULT_CHUNK_SIZE,
         **kwargs
     ):
         super().__init__(entity_type, filetype, **kwargs)
-        self._workspace_id = workspace_id
-        self._entity_id = entity_id
-        self._client = client
-        self._chunk_size = chunk_size
-        
         self._buffer: Optional[BytesIO] = None
         self._fully_loaded = False
         self._modified = False
         self._position = 0
         self._size: Optional[int] = None
-        self._presigned_url: Optional[str] = None
+        self._download_url: Optional[str] = None
         self._http_client: Optional[httpx.Client] = None
-        
-        if initial_data is not None:
-            self._buffer = BytesIO(initial_data)
-            self._fully_loaded = True
-            self._size = len(initial_data)
     
-    def _get_presigned_url(self) -> str:
-        """Get or refresh the presigned URL for the attachment."""
-        from octostar.utils.workspace import get_attachment_url
+    # ==================== Abstract URL methods ====================
+    
+    @abstractmethod
+    def _refresh_download_url(self) -> str:
+        """Fetch a fresh presigned download URL from the API."""
+        pass
+    
+    @abstractmethod
+    def _get_upload_info(self) -> Dict[str, Any]:
+        """
+        Get presigned upload URL and form fields from the API.
         
-        return get_attachment_url.sync(
-            os_workspace=self._workspace_id,
-            os_entity_uid=self._entity_id,
-            client=self._client
-        )
+        Returns:
+            {"url": "<presigned upload URL>", "fields": {<form fields>}}
+        """
+        pass
+    
+    # ==================== HTTP infrastructure ====================
     
     def _ensure_http_client(self):
-        """Ensure HTTP client is initialized."""
+        """Ensure the shared httpx.Client is initialized."""
         if not self._http_client:
             self._http_client = httpx.Client(timeout=self.DEFAULT_URL_TIMEOUT)
     
+    def _ensure_download_url(self):
+        """Ensure a download URL is cached, fetching one if needed."""
+        if not self._download_url:
+            self._download_url = self._refresh_download_url()
+    
+    # ==================== Remote read operations ====================
+    
     def _fetch_size(self) -> int:
         """
-        Fetch the size of the remote attachment using HEAD request.
+        Fetch the size of the remote content using a HEAD request.
         
         Handles presigned URL expiration with automatic refresh and retry.
         """
         if self._size is not None:
             return self._size
         
-        if not self._presigned_url:
-            self._presigned_url = self._get_presigned_url()
-        
+        self._ensure_download_url()
         self._ensure_http_client()
-        max_retries = 3
         
-        for attempt in range(max_retries):
-            response = self._http_client.head(self._presigned_url)
+        for attempt in range(self.MAX_RETRIES):
+            response = self._http_client.head(self._download_url)
             if response.status_code == 200:
                 self._size = int(response.headers.get('content-length', 0))
                 return self._size
             elif response.status_code == 403:
                 _logger.debug(
-                    f"Presigned URL expired while fetching size, refreshing... (attempt {attempt + 1}/{max_retries})"
+                    f"Presigned URL expired while fetching size, refreshing... "
+                    f"(attempt {attempt + 1}/{self.MAX_RETRIES})"
                 )
-                self._presigned_url = self._get_presigned_url()
+                self._download_url = self._refresh_download_url()
                 continue
             else:
                 response.raise_for_status()
 
         raise ConnectionError(
-            f"Failed to fetch file size after {max_retries} attempts (URL kept expiring)"
+            f"Failed to fetch file size after {self.MAX_RETRIES} attempts (URL kept expiring)"
         )
     
     def _read_range(self, start: int, end: int) -> bytes:
         """
-        Read a specific byte range using HTTP Range request.
+        Read a specific byte range using an HTTP Range request.
         
         Handles presigned URL expiration with automatic refresh and retry.
         
@@ -526,16 +525,13 @@ class WorkspaceAttachmentContents(Contents):
         Returns:
             Bytes from the specified range
         """
-        if not self._presigned_url:
-            self._presigned_url = self._get_presigned_url()
-        
+        self._ensure_download_url()
         self._ensure_http_client()
         
         headers = {"Range": f"bytes={start}-{end}"}
-        max_retries = 3
         
-        for attempt in range(max_retries):
-            response = self._http_client.get(self._presigned_url, headers=headers)
+        for attempt in range(self.MAX_RETRIES):
+            response = self._http_client.get(self._download_url, headers=headers)
             
             if response.status_code == 206:
                 return response.content
@@ -543,38 +539,85 @@ class WorkspaceAttachmentContents(Contents):
                 return b""
             elif response.status_code == 403:
                 _logger.debug(
-                    f"Presigned URL expired at byte {start}, refreshing... (attempt {attempt + 1}/{max_retries})"
+                    f"Presigned URL expired at byte {start}, refreshing... "
+                    f"(attempt {attempt + 1}/{self.MAX_RETRIES})"
                 )
-                self._presigned_url = self._get_presigned_url()
+                self._download_url = self._refresh_download_url()
                 continue
             else:
                 response.raise_for_status()
                 return response.content
                 
         raise ConnectionError(
-            f"Failed to read range {start}-{end} after {max_retries} attempts (URL kept expiring)"
+            f"Failed to read range {start}-{end} after {self.MAX_RETRIES} attempts "
+            f"(URL kept expiring)"
         )
     
     def _load_full(self):
-        """Load the entire attachment into memory buffer using read_attachment()."""
+        """
+        Load the entire remote content into memory buffer via presigned URL.
+        
+        Handles presigned URL expiration with automatic refresh and retry.
+        After loading, the buffer position is restored to self._position.
+        """
         if self._fully_loaded:
             return
         
-        from octostar.utils.workspace import read_attachment
+        self._ensure_download_url()
+        self._ensure_http_client()
         
-        data = read_attachment.sync(
-            os_workspace=self._workspace_id,
-            os_entity_uid=self._entity_id,
-            decode=False,
-            stream=False,
-            client=self._client
+        for attempt in range(self.MAX_RETRIES):
+            response = self._http_client.get(self._download_url)
+            if response.status_code == 200:
+                data = response.content
+                self._buffer = BytesIO(data)
+                self._fully_loaded = True
+                self._size = len(data)
+                self._buffer.seek(self._position, SEEK_SET)
+                return
+            elif response.status_code == 403:
+                _logger.debug(
+                    f"Presigned URL expired during full load, refreshing... "
+                    f"(attempt {attempt + 1}/{self.MAX_RETRIES})"
+                )
+                self._download_url = self._refresh_download_url()
+                continue
+            else:
+                response.raise_for_status()
+        
+        raise ConnectionError(
+            f"Failed to load full content after {self.MAX_RETRIES} attempts "
+            f"(URL kept expiring)"
         )
-        self._buffer = BytesIO(data or b"")
-        self._fully_loaded = True
-        self._size = len(data) if data else 0
-        self._position = 0
+    
+    # ==================== Remote write operations ====================
+    
+    def _upload_data(self, data: bytes):
+        """
+        Upload data to remote storage via presigned POST URL.
+        
+        Uses the presigned URL and form fields from _get_upload_info().
+        This only touches S3/MinIO — no entity record upsert.
+        """
+        upload_info = self._get_upload_info()
+        upload_url = upload_info["url"]
+        upload_fields = upload_info.get("fields", {})
+        
+        self._ensure_http_client()
+        file_obj = BytesIO(data)
+        
+        response = self._http_client.post(
+            upload_url,
+            data=upload_fields,
+            files={"file": file_obj},
+        )
+        if response.status_code not in (200, 204):
+            response.raise_for_status()
+    
+    # ==================== FileIO Interface ====================
     
     def read(self, size: int = -1) -> bytes:
+        # If fully loaded into buffer, read from buffer
         if self._fully_loaded:
             if not self._buffer:
                 self._load_full()
@@ -582,7 +625,6 @@ class WorkspaceAttachmentContents(Contents):
         
         # Otherwise, use HTTP Range requests for efficient streaming
         if size == -1:
-            # Read all from current position
             self._fetch_size()
             if self._position >= self._size:
                 return b""
@@ -591,33 +633,28 @@ class WorkspaceAttachmentContents(Contents):
         if size <= 0:
             return b""
         
-        # Fetch the size if we don't know it
         self._fetch_size()
         
-        # Adjust size if it goes beyond the end
+        # Clamp to available bytes
         if self._position + size > self._size:
             size = self._size - self._position
         
         if size <= 0:
             return b""
         
-        # Read using HTTP Range request
         end_byte = self._position + size - 1
         data = self._read_range(self._position, end_byte)
         self._position += len(data)
-        
         return data
     
     def write(self, b: bytes) -> int:
         if not self._buffer:
             self._load_full()
-        
         n = self._buffer.write(b)
         self._modified = True
         return n
     
     def seek(self, offset: int, whence: int = SEEK_SET) -> int:
-        # Calculate new position
         if whence == SEEK_SET:
             new_pos = offset
         elif whence == SEEK_CUR:
@@ -634,11 +671,9 @@ class WorkspaceAttachmentContents(Contents):
         if new_pos < 0:
             raise ValueError("Negative seek position")
         
-        # If fully loaded, use buffer seek
         if self._buffer and self._fully_loaded:
             return self._buffer.seek(new_pos, SEEK_SET)
         
-        # Otherwise, just update position (HTTP Range will handle it)
         self._position = new_pos
         return self._position
     
@@ -648,37 +683,30 @@ class WorkspaceAttachmentContents(Contents):
         return self._position
     
     def flush(self):
-        """Flush the internal buffer and write to workspace if modified."""
+        """Flush: upload buffer to remote storage via presigned URL, then invalidate cache."""
         if self._buffer:
             self._buffer.flush()
         
         if not self._modified or not self._buffer:
             return
         
-        if not self._entity_type or not self._filetype:
-            raise ValueError("entity_type and filetype required to flush to workspace")
-        
-        from octostar.utils.workspace import write_attachment
-        
+        # Read all data from buffer
         current_pos = self._buffer.tell()
         self._buffer.seek(0, SEEK_SET)
         data = self._buffer.read()
         self._buffer.seek(current_pos, SEEK_SET)
         
-        write_attachment.sync(
-            os_workspace=self._workspace_id,
-            os_entity_uid=self._entity_id,
-            entity_type=self._entity_type,
-            filetype=self._filetype,
-            file=data,
-            client=self._client
-        )
+        # Upload directly to S3 via presigned URL
+        self._upload_data(data)
+        
+        # Invalidate cache so next read re-fetches from remote
         self._modified = False
         self._buffer.close()
         self._buffer = None
         self._fully_loaded = False
         self._position = 0
-        self._presigned_url = None  # force refresh on next remote read
+        self._size = None
+        self._download_url = None
     
     def close(self):
         if not self._closed:
@@ -691,15 +719,6 @@ class WorkspaceAttachmentContents(Contents):
                 self._http_client = None
             super().close()
     
-    def delete(self):
-        """Delete the entity from the workspace using delete_entity()."""
-        from octostar.utils.workspace import delete_entity
-        
-        delete_entity.sync(
-            os_entity_uid=self._entity_id,
-            client=self._client
-        )
-    
     def truncate(self, size: Optional[int] = None) -> int:
         if not self._buffer:
             self._load_full()
@@ -710,6 +729,64 @@ class WorkspaceAttachmentContents(Contents):
         if not self._buffer or not self._fully_loaded:
             self._load_full()
         return self._buffer.getvalue()
+
+
+class WorkspaceAttachmentContents(_PresignedUrlRemoteContents):
+    """
+    Contents implementation for Octostar workspace attachments.
+    
+    All I/O goes through presigned S3 URLs:
+    - Reads use get_attachment_url() for presigned download URLs
+    - Writes use request_attachment_url() for presigned upload URLs
+    - No entity upsert on flush — only raw bytes are written to S3
+    
+    Presigned URL Handling:
+    - Download URLs are obtained via get_attachment_url() and cached
+    - On 403 (Forbidden) responses, URLs are refreshed and requests retried
+    """
+    
+    def __init__(
+        self,
+        entity_type: Optional[str] = None,
+        filetype: Optional[str] = None,
+        *,
+        workspace_id: str,
+        entity_id: str,
+        client,
+        **kwargs
+    ):
+        super().__init__(entity_type, filetype, **kwargs)
+        self._workspace_id = workspace_id
+        self._entity_id = entity_id
+        self._client = client
+    
+    def _refresh_download_url(self) -> str:
+        from octostar.utils.workspace import get_attachment_url
+        
+        return get_attachment_url.sync(
+            os_workspace=self._workspace_id,
+            os_entity_uid=self._entity_id,
+            client=self._client
+        )
+    
+    def _get_upload_info(self) -> Dict[str, Any]:
+        from octostar.utils.workspace import request_attachment_url
+        
+        return request_attachment_url.sync(
+            os_workspace=self._workspace_id,
+            os_entity_uid=self._entity_id,
+            content_type=self._filetype,
+            client=self._client
+        )
+    
+    def delete(self):
+        """Delete the entity from the workspace using delete_entity()."""
+        from octostar.utils.workspace import delete_entity
+        
+        delete_entity.sync(
+            os_entity_uid=self._entity_id,
+            client=self._client
+        )
     
     def to_locator(self) -> Dict[str, Any]:
         """
@@ -717,7 +794,7 @@ class WorkspaceAttachmentContents(Contents):
         
         Returns:
             {"location": "workspace_attachment", "pointer": "workspace_id/entity_id", 
-             "entity_type": "...", "filetype": "..."}
+             "entity_type": "...", "filetype": "...", "item_name": "..."}
         """
         if self._workspace_id and self._entity_id:
             pointer = f"{self._workspace_id}/{self._entity_id}"
@@ -765,12 +842,13 @@ class WorkspaceAttachmentContents(Contents):
         )
 
 
-class TemporaryAttachmentContents(Contents):
+class TemporaryAttachmentContents(_PresignedUrlRemoteContents):
     """
     Contents implementation for Octostar temporary blob storage.
     
-    Uses octostar-api utilities (read_temporary_blob, write_temporary_blob,
-    delete_temporary_blob) to store files in the user's temporary S3 bucket.
+    All I/O goes through presigned S3 URLs:
+    - Reads use get_temporary_blob_url() for presigned download URLs
+    - Writes use request_temporary_blob_url() for presigned upload URLs
     
     Temporary blobs are keyed by filename (not workspace/entity), and are not
     associated with any workspace entity. Use WorkspaceAttachmentContents for that.
@@ -783,90 +861,27 @@ class TemporaryAttachmentContents(Contents):
         *,
         filename: str,
         client,
-        initial_data: Optional[bytes] = None,
         **kwargs
     ):
         super().__init__(entity_type, filetype, **kwargs)
         self._filename = filename
         self._client = client
-        
-        self._buffer: Optional[BytesIO] = None
-        self._fully_loaded = False
-        self._modified = False
-        
-        if initial_data is not None:
-            self._buffer = BytesIO(initial_data)
-            self._fully_loaded = True
     
-    def _load_full(self):
-        """Load the entire blob into memory using read_temporary_blob()."""
-        if self._fully_loaded:
-            return
+    def _refresh_download_url(self) -> str:
+        from octostar.utils.workspace import get_temporary_blob_url
         
-        from octostar.utils.workspace import read_temporary_blob
-        
-        data = read_temporary_blob.sync(
+        return get_temporary_blob_url.sync(
             filename=self._filename,
-            decode=False,
             client=self._client
         )
-        self._buffer = BytesIO(data or b"")
-        self._fully_loaded = True
-    
-    def read(self, size: int = -1) -> bytes:
-        if not self._buffer:
-            self._load_full()
-        return self._buffer.read(size)
-    
-    def write(self, b: bytes) -> int:
-        if not self._buffer:
-            self._load_full()
-        n = self._buffer.write(b)
-        self._modified = True
-        return n
     
-    def seek(self, offset: int, whence: int = SEEK_SET) -> int:
-        if not self._buffer:
-            self._load_full()
-        return self._buffer.seek(offset, whence)
-    
-    def tell(self) -> int:
-        if not self._buffer:
-            self._load_full()
-        return self._buffer.tell()
-    
-    def flush(self):
-        """Flush the internal buffer and write to temp bucket if modified."""
-        if self._buffer:
-            self._buffer.flush()
-        
-        if not self._modified or not self._buffer:
-            return
-        
-        from octostar.utils.workspace import write_temporary_blob
+    def _get_upload_info(self) -> Dict[str, Any]:
+        from octostar.utils.workspace import request_temporary_blob_url
         
-        current_pos = self._buffer.tell()
-        self._buffer.seek(0, SEEK_SET)
-        data = self._buffer.read()
-        self._buffer.seek(current_pos, SEEK_SET)
-        
-        write_temporary_blob.sync(
+        return request_temporary_blob_url.sync(
             filename=self._filename,
-            file=data,
             client=self._client
         )
-        self._modified = False
-        self._buffer.close()
-        self._buffer = None
-        self._fully_loaded = False
-    
-    def close(self):
-        if not self._closed:
-            if self._modified:
-                self.flush()
-            if self._buffer:
-                self._buffer.close()
-            super().close()
     
     def delete(self):
         """Delete the blob from the temporary bucket."""
@@ -877,17 +892,6 @@ class TemporaryAttachmentContents(Contents):
             client=self._client
         )
     
-    def truncate(self, size: Optional[int] = None) -> int:
-        if not self._buffer:
-            self._load_full()
-        self._modified = True
-        return self._buffer.truncate(size)
-    
-    def getvalue(self) -> bytes:
-        if not self._buffer or not self._fully_loaded:
-            self._load_full()
-        return self._buffer.getvalue()
-    
     def to_locator(self) -> Dict[str, Any]:
         """
         Serialize to locator with filename.

{streamlit_octostar_utils-0.4.2.dev9 → streamlit_octostar_utils-0.4.2.dev10}/streamlit_octostar_utils/api_crafter/nifi.py

@@ -18,7 +18,7 @@ from octostar.utils.workspace import upsert_entities
 from octostar.utils.ontology import fetch_ontology_data
 from octostar.utils.workspace.permissions import get_permissions, PermissionLevel
 from octostar.utils.pipeline import update_processing_status
-from octostar.utils.workspace import write_attachment
+
 from octostar.client import make_client
 
 from ..core.dict import recursive_update_dict, travel_dict, jsondict_hash
@@ -464,26 +464,49 @@ class NifiContextManager(object):
             if not file.contents:
                 continue
             old_contents = file._contents
-            write_attachment.sync(
-                os_workspace=file.write_os_workspace,
-                os_entity_uid=file.record["os_entity_uid"],
-                entity_type=file.record["os_concept"],
-                filetype=file.record["os_item_content_type"],
-                file=file.contents,
-                client=self.client,
-            )
-            file._contents = WorkspaceAttachmentContents(
-                workspace_id=file.record['os_workspace'],
-                entity_id=file.record['os_entity_uid'],
+            record = file.record
+            ws = file.write_os_workspace
+            entity_uid = record["os_entity_uid"]
+            entity_type = record["os_concept"]
+            filetype = record.get("os_item_content_type")
+            item_name = record.get("os_item_name")
+            data = old_contents.getvalue()
+            target = WorkspaceAttachmentContents(
+                workspace_id=ws,
+                entity_id=entity_uid,
                 client=self.client,
-                entity_type=file.record["os_concept"],
-                filetype=file.record["os_item_content_type"]
+                entity_type=entity_type,
+                filetype=filetype,
             )
+            target.write(data)
+            target.flush()
+            file._contents = target
             if isinstance(old_contents, TemporaryAttachmentContents):
                 try:
                     old_contents.delete()
                 except Exception:
                     pass
+            fields = {"os_has_attachment": True}
+            if filetype:
+                fields["os_item_content_type"] = filetype
+            if item_name:
+                fields["os_item_name"] = item_name
+            new_entities = upsert_entities.sync(
+                ws,
+                [{
+                    "entity_type": entity_type,
+                    "os_entity_uid": entity_uid,
+                    "fields": fields,
+                }],
+                client=self.client,
+            )
+            new_entity = {e["os_entity_uid"]: e for e in new_entities}.get(entity_uid, {})
+            file.record = {**record, **new_entity}
+            file.record["entity_id"] = file.record["os_entity_uid"]
+            file.record["entity_type"] = file.record["os_concept"]
+            file.record["entity_label"] = file.label
+            file.request["entity_timestamp"] = file.record["os_last_updated_at"]
+            file.request["is_temporary"] = False
 
     def _sync_upsert_entities(self, entities_to_upsert):
         if not entities_to_upsert: