figpack 0.2.27__py3-none-any.whl → 0.2.40__py3-none-any.whl

figpack/core/_upload_bundle.py

@@ -1,3 +1,4 @@
+from typing import Optional, Union
 import hashlib
 import json
 import pathlib
@@ -114,62 +115,31 @@ def _upload_single_file_with_signed_url(
         else:
             break
 
+    assert last_exception is not None
     raise last_exception
 
 
 MAX_WORKERS_FOR_UPLOAD = 16
 
 
-def _compute_deterministic_figure_hash(tmpdir_path: pathlib.Path) -> str:
-    """
-    Compute a deterministic figure ID based on SHA1 hashes of all files
-
-    Returns:
-        str: 40-character SHA1 hash representing the content of all files
-    """
-    file_hashes = []
-
-    # Collect all files and their hashes
-    for file_path in sorted(tmpdir_path.rglob("*")):
-        if file_path.is_file():
-            relative_path = file_path.relative_to(tmpdir_path)
-
-            # Compute SHA1 hash of file content
-            sha1_hash = hashlib.sha1()
-            with open(file_path, "rb") as f:
-                for chunk in iter(lambda: f.read(4096), b""):
-                    sha1_hash.update(chunk)
-
-            # Include both the relative path and content hash to ensure uniqueness
-            file_info = f"{relative_path}:{sha1_hash.hexdigest()}"
-            file_hashes.append(file_info)
-
-    # Create final hash from all file hashes
-    combined_hash = hashlib.sha1()
-    for file_hash in file_hashes:
-        combined_hash.update(file_hash.encode("utf-8"))
-
-    return combined_hash.hexdigest()
-
-
 def _create_or_get_figure(
-    figure_hash: str,
-    api_key: str,
-    total_files: int = None,
-    total_size: int = None,
-    title: str = None,
+    api_key: Optional[str],
+    total_files: Optional[int] = None,
+    total_size: Optional[int] = None,
+    title: Optional[str] = None,
     ephemeral: bool = False,
+    source_url: Optional[str] = None,
 ) -> dict:
     """
     Create a new figure or get existing figure information
 
     Args:
-        figure_hash: The hash of the figure
         api_key: The API key for authentication (required for non-ephemeral)
         total_files: Optional total number of files
         total_size: Optional total size of files
         title: Optional title for the figure
         ephemeral: Whether to create an ephemeral figure
+        source_url: Optional source URL for the figure (must be unique)
 
     Returns:
         dict: Figure information from the API
@@ -178,8 +148,7 @@ def _create_or_get_figure(
     if not ephemeral and api_key is None:
         raise ValueError("API key is required for non-ephemeral figures")
 
-    payload = {
-        "figureHash": figure_hash,
+    payload: dict[str, Union[str, int]] = {
         "figpackVersion": __version__,
         "bucket": FIGPACK_BUCKET,
     }
@@ -196,6 +165,8 @@ def _create_or_get_figure(
         payload["title"] = title
     if ephemeral:
         payload["ephemeral"] = True
+    if source_url is not None:
+        payload["sourceUrl"] = source_url
 
     # Use the same endpoint for both regular and ephemeral figures
     response = requests.post(f"{FIGPACK_API_BASE_URL}/api/figures/create", json=payload)
@@ -206,12 +177,12 @@ def _create_or_get_figure(
             error_msg = error_data.get("message", "Unknown error")
         except:
             error_msg = f"HTTP {response.status_code}"
-        raise Exception(f"Failed to create figure {figure_hash}: {error_msg}")
+        raise Exception(f"Failed to create figure: {error_msg}")
 
     response_data = response.json()
     if not response_data.get("success"):
         raise Exception(
-            f"Failed to create figure {figure_hash}: {response_data.get('message', 'Unknown error')}"
+            f"Failed to create figure: {response_data.get('message', 'Unknown error')}"
         )
 
     return response_data
@@ -252,10 +223,11 @@ def _finalize_figure(figure_url: str, api_key: str) -> dict:
 
 def _upload_bundle(
     tmpdir: str,
-    api_key: str,
-    title: str = None,
+    api_key: Optional[str],
+    title: Optional[str] = None,
     ephemeral: bool = False,
     use_consolidated_metadata_only: bool = False,
+    source_url: Optional[str] = None,
 ) -> str:
     """
     Upload the prepared bundle to the cloud using the new database-driven approach
@@ -267,12 +239,10 @@ def _upload_bundle(
         ephemeral: Whether to create an ephemeral figure
         use_consolidated_metadata_only: If True, excludes individual zarr metadata files
             (.zgroup, .zarray, .zattrs) since they are included in .zmetadata
+        source_url: Optional source URL for the figure (must be unique)
     """
     tmpdir_path = pathlib.Path(tmpdir)
 
-    # Compute deterministic figure ID based on file contents
-    figure_hash = _compute_deterministic_figure_hash(tmpdir_path)
-
     # Collect all files to upload
     all_files = []
     for file_path in tmpdir_path.rglob("*"):
@@ -293,7 +263,12 @@ def _upload_bundle(
 
     # Find available figure ID and create/get figure in database with metadata
     result = _create_or_get_figure(
-        figure_hash, api_key, total_files, total_size, title=title, ephemeral=ephemeral
+        api_key,
+        total_files,
+        total_size,
+        title=title,
+        ephemeral=ephemeral,
+        source_url=source_url,
     )
     figure_info = result.get("figure", {})
     figure_url = figure_info.get("figureUrl")
@@ -329,7 +304,9 @@ def _upload_bundle(
 
             # Get signed URLs for this batch
             try:
-                signed_urls_map = _get_batch_signed_urls(figure_url, batch, api_key)
+                signed_urls_map = _get_batch_signed_urls(
+                    figure_url, batch, api_key if api_key else ""
+                )
             except Exception as e:
                 print(f"Failed to get signed URLs for batch {batch_num}: {e}")
                 raise
@@ -400,7 +377,9 @@ def _upload_bundle(
     try:
         # Use batch API for manifest
         manifest_batch = [("manifest.json", temp_file_path)]
-        signed_urls_map = _get_batch_signed_urls(figure_url, manifest_batch, api_key)
+        signed_urls_map = _get_batch_signed_urls(
+            figure_url, manifest_batch, api_key if api_key else ""
+        )
 
         if "manifest.json" not in signed_urls_map:
             raise Exception("No signed URL returned for manifest.json")
@@ -418,12 +397,45 @@ def _upload_bundle(
 
     # Finalize the figure upload
     print("Finalizing figure...")
-    _finalize_figure(figure_url, api_key)
+    _finalize_figure(figure_url, api_key if api_key else "")
     print("Upload completed successfully")
 
     return figure_url
 
 
+def get_figure_by_source_url(source_url: str) -> Optional[str]:
+    """
+    Query the API for a figure URL by its source URL
+
+    Args:
+        source_url: The source URL to search for
+
+    Returns:
+        Optional[str]: The figure URL if found, None otherwise
+    """
+    payload = {"sourceUrl": source_url}
+
+    response = requests.post(
+        f"{FIGPACK_API_BASE_URL}/api/figures/find-by-source-url", json=payload
+    )
+
+    if not response.ok:
+        if response.status_code == 404:
+            return None
+        try:
+            error_data = response.json()
+            error_msg = error_data.get("message", "Unknown error")
+        except:
+            error_msg = f"HTTP {response.status_code}"
+        raise Exception(f"Failed to query figure by source URL: {error_msg}")
+
+    response_data = response.json()
+    if not response_data.get("success"):
+        return None
+
+    return response_data.get("figureUrl")
+
+
 def _determine_content_type(file_path: str) -> str:
     """
     Determine content type for upload based on file extension

figpack/core/_view_figure.py

@@ -34,10 +34,10 @@ def serve_files(
         enable_file_upload: Whether to enable PUT requests for file uploads
         max_file_size: Maximum file size in bytes for uploads (default 10MB)
     """
-    tmpdir = pathlib.Path(tmpdir)
-    tmpdir = tmpdir.resolve()
-    if not tmpdir.exists() or not tmpdir.is_dir():
-        raise SystemExit(f"Directory not found: {tmpdir}")
+    tmpdir_2 = pathlib.Path(tmpdir)
+    tmpdir_2 = tmpdir_2.resolve()
+    if not tmpdir_2.exists() or not tmpdir_2.is_dir():
+        raise SystemExit(f"Directory not found: {tmpdir_2}")
 
     # Create a temporary server manager instance for this specific directory
     # Note: We can't use the singleton ProcessServerManager here because it serves
@@ -56,29 +56,34 @@ def serve_files(
     # Choose handler based on file upload requirement
     if enable_file_upload:
 
-        def handler_factory(*args, **kwargs):
+        def handler_factory_upload_enabled(*args, **kwargs):
             return FileUploadCORSRequestHandler(
                 *args,
-                directory=str(tmpdir),
+                directory=str(tmpdir_2),
                 allow_origin=allow_origin,
                 enable_file_upload=True,
                 max_file_size=max_file_size,
                 **kwargs,
             )
 
-        upload_status = " (file upload enabled)" if enable_file_upload else ""
+        upload_status = (
+            " (file upload enabled)" if handler_factory_upload_enabled else ""
+        )
+
+        httpd = ThreadingHTTPServer(("0.0.0.0", port), handler_factory_upload_enabled)  # type: ignore
     else:
 
         def handler_factory(*args, **kwargs):
             return CORSRequestHandler(
-                *args, directory=str(tmpdir), allow_origin=allow_origin, **kwargs
+                *args, directory=str(tmpdir_2), allow_origin=allow_origin, **kwargs
             )
 
         upload_status = ""
 
-    httpd = ThreadingHTTPServer(("0.0.0.0", port), handler_factory)
+        httpd = ThreadingHTTPServer(("0.0.0.0", port), handler_factory)  # type: ignore
+
     print(
-        f"Serving {tmpdir} at http://localhost:{port} (CORS → {allow_origin}){upload_status}"
+        f"Serving {tmpdir_2} at http://localhost:{port} (CORS → {allow_origin}){upload_status}"
     )
     thread = threading.Thread(target=httpd.serve_forever, daemon=True)
     thread.start()

figpack/core/_zarr_consolidate.py

@@ -0,0 +1,185 @@
+import os
+import pathlib
+import json
+from typing import Dict, List, Tuple
+
+
+def consolidate_zarr_chunks(
+    zarr_dir: pathlib.Path, max_file_size: int = 100_000_000
+) -> None:
+    """
+    Consolidate zarr chunk files into larger files to reduce the number of files
+    that need to be uploaded. Updates the .zmetadata file with refs mapping.
+
+    Args:
+        zarr_dir: Path to the zarr directory
+        max_file_size: Maximum size for each consolidated file in bytes (default: 100 MB)
+    """
+    if not zarr_dir.is_dir():
+        raise ValueError(f"Expected a directory, got: {zarr_dir}")
+
+    # Read the existing .zmetadata file
+    zmetadata_path = zarr_dir / ".zmetadata"
+    if not zmetadata_path.exists():
+        raise ValueError(f"No .zmetadata file found at {zmetadata_path}")
+
+    with open(zmetadata_path, "r") as f:
+        zmetadata = json.load(f)
+
+    # Collect all chunk files (non-metadata files)
+    chunk_files = _collect_chunk_files(zarr_dir)
+
+    if not chunk_files:
+        # No chunk files to consolidate
+        return
+
+    # Group chunk files into consolidated files
+    consolidated_groups = _group_files_by_size(chunk_files, max_file_size)
+
+    # Create consolidated files and build refs mapping
+    refs: Dict[str, List] = {}
+    for group_idx, file_group in enumerate(consolidated_groups):
+        consolidated_filename = f"_consolidated_{group_idx}.dat"
+        consolidated_path = zarr_dir / consolidated_filename
+
+        # Write the consolidated file and track byte offsets
+        current_offset = 0
+        with open(consolidated_path, "wb") as consolidated_file:
+            for file_path, relative_path in file_group:
+                # Read the chunk file
+                with open(file_path, "rb") as chunk_file:
+                    chunk_data = chunk_file.read()
+
+                # Write to consolidated file
+                consolidated_file.write(chunk_data)
+
+                # Add to refs mapping
+                refs[relative_path] = [
+                    consolidated_filename,
+                    current_offset,
+                    len(chunk_data),
+                ]
+
+                # Update offset
+                current_offset += len(chunk_data)
+
+    # Update .zmetadata with refs
+    zmetadata["refs"] = refs
+
+    # Write updated .zmetadata
+    with open(zmetadata_path, "w") as f:
+        json.dump(zmetadata, f, indent=2)
+
+    # Delete original chunk files
+    for file_path, _ in chunk_files:
+        try:
+            file_path.unlink()
+        except Exception as e:
+            print(f"Warning: could not remove file {file_path}: {e}")
+
+    # Clean up empty directories
+    _remove_empty_directories(zarr_dir)
+
+
+def _collect_chunk_files(zarr_dir: pathlib.Path) -> List[Tuple[pathlib.Path, str]]:
+    """
+    Collect all chunk files in the zarr directory (excluding metadata files).
+
+    Args:
+        zarr_dir: Path to the zarr directory
+
+    Returns:
+        List of tuples (absolute_path, relative_path) for each chunk file
+    """
+    chunk_files = []
+    metadata_files = {".zmetadata", ".zarray", ".zgroup", ".zattrs"}
+
+    for root, dirs, files in os.walk(zarr_dir):
+        for file in files:
+            # Skip metadata files
+            if file in metadata_files or file.startswith("_consolidated_"):
+                continue
+
+            file_path = pathlib.Path(root) / file
+            # Get relative path from zarr_dir
+            relative_path = file_path.relative_to(zarr_dir).as_posix()
+
+            chunk_files.append((file_path, relative_path))
+
+    return chunk_files
+
+
+def _group_files_by_size(
+    files: List[Tuple[pathlib.Path, str]], max_size: int
+) -> List[List[Tuple[pathlib.Path, str]]]:
+    """
+    Group files into bins where each bin's total size is <= max_size.
+
+    Uses a simple first-fit bin packing algorithm.
+
+    Args:
+        files: List of (file_path, relative_path) tuples
+        max_size: Maximum total size for each group in bytes
+
+    Returns:
+        List of groups, where each group is a list of (file_path, relative_path) tuples
+    """
+    # Get file sizes
+    files_with_sizes = []
+    for file_path, relative_path in files:
+        try:
+            size = file_path.stat().st_size
+            files_with_sizes.append((file_path, relative_path, size))
+        except Exception as e:
+            print(f"Warning: could not get size of {file_path}: {e}")
+            continue
+
+    # Sort by size (largest first) for better packing
+    files_with_sizes.sort(key=lambda x: x[2], reverse=True)
+
+    # First-fit bin packing
+    groups: List[List[Tuple[pathlib.Path, str]]] = []
+    group_sizes: List[int] = []
+
+    for file_path, relative_path, size in files_with_sizes:
+        # If file is larger than max_size, put it in its own group
+        if size > max_size:
+            groups.append([(file_path, relative_path)])
+            group_sizes.append(size)
+            continue
+
+        # Try to fit into existing group
+        placed = False
+        for i, group_size in enumerate(group_sizes):
+            if group_size + size <= max_size:
+                groups[i].append((file_path, relative_path))
+                group_sizes[i] += size
+                placed = True
+                break
+
+        # If doesn't fit anywhere, create new group
+        if not placed:
+            groups.append([(file_path, relative_path)])
+            group_sizes.append(size)
+
+    return groups
+
+
+def _remove_empty_directories(zarr_dir: pathlib.Path) -> None:
+    """
+    Remove empty directories within the zarr directory.
+
+    Args:
+        zarr_dir: Path to the zarr directory
+    """
+    # Walk bottom-up so we can remove empty parent directories
+    for root, dirs, files in os.walk(zarr_dir, topdown=False):
+        for dir_name in dirs:
+            dir_path = pathlib.Path(root) / dir_name
+            try:
+                # Only remove if directory is empty
+                if not any(dir_path.iterdir()):
+                    dir_path.rmdir()
+            except Exception:
+                # Directory not empty or other error, skip
+                pass

figpack/core/extension_view.py

@@ -2,9 +2,13 @@
 Base class for views that use figpack extensions
 """
 
+from typing import TYPE_CHECKING
+
 from .figpack_view import FigpackView
-from .figpack_extension import FigpackExtension
-from ..core.zarr import Group
+
+if TYPE_CHECKING:
+    from .figpack_extension import FigpackExtension
+    from .zarr import Group
 
 
 class ExtensionView(FigpackView):
@@ -12,7 +16,7 @@ class ExtensionView(FigpackView):
     Base class for views that are rendered by figpack extensions
     """
 
-    def __init__(self, *, extension: FigpackExtension, view_type: str):
+    def __init__(self, *, extension: "FigpackExtension", view_type: str) -> None:
         """
         Initialize an extension-based view
 
@@ -23,7 +27,7 @@ class ExtensionView(FigpackView):
         self.extension = extension
         self.view_type = view_type
 
-    def write_to_zarr_group(self, group: Group) -> None:
+    def write_to_zarr_group(self, group: "Group") -> None:
         """
         Write the extension view metadata to a Zarr group.
         Subclasses should call super().write_to_zarr_group(group) first,

figpack/core/figpack_extension.py

@@ -17,7 +17,7 @@ class FigpackExtension:
         javascript_code: str,
         additional_files: Optional[Dict[str, str]] = None,
         version: str = "1.0.0",
-    ):
+    ) -> None:
         """
         Initialize a figpack extension
 

figpack/core/figpack_view.py

@@ -5,7 +5,7 @@ Base view class for figpack visualization components
 import os
 import random
 import string
-from typing import Union
+from typing import Optional
 
 from .zarr import Group
 
@@ -19,17 +19,17 @@ class FigpackView:
         self,
         *,
         title: str,
-        description: Union[str, None] = None,
-        port: Union[int, None] = None,
-        open_in_browser: Union[bool, None] = None,
-        upload: Union[bool, None] = None,
-        inline: Union[bool, None] = None,
+        description: Optional[str] = None,
+        port: Optional[int] = None,
+        open_in_browser: Optional[bool] = None,
+        upload: Optional[bool] = None,
+        inline: Optional[bool] = None,
         inline_height: int = 600,
-        ephemeral: Union[bool, None] = None,
-        allow_origin: Union[str, None] = None,
-        wait_for_input: Union[bool, None] = None,
-        _dev: Union[bool, None] = None,
-    ):
+        ephemeral: Optional[bool] = None,
+        allow_origin: Optional[str] = None,
+        wait_for_input: Optional[bool] = None,
+        _dev: Optional[bool] = None,
+    ) -> None:
         """
         Display a figpack view component with intelligent environment detection and flexible display options.
         See https://flatironinstitute.github.io/figpack/show_function.html for complete documentation.
@@ -86,6 +86,8 @@ class FigpackView:
                 inline = False
             elif _is_in_notebook() and not upload:
                 inline = True
+            else:
+                inline = False
 
         # determine open_in_browser
         if open_in_browser is None:
@@ -111,10 +113,22 @@ class FigpackView:
                     upload = True
                     ephemeral = True
 
+        if ephemeral is None:
+            ephemeral = False
+
+        if upload is None:
+            upload = False
+
         # determine _dev
         if _dev is None:
             _dev = os.environ.get("FIGPACK_DEV") == "1"
 
+        if port is None and os.environ.get("FIGPACK_PORT"):
+            try:
+                port = int(os.environ.get("FIGPACK_PORT", ""))
+            except Exception:
+                pass
+
         # determine wait_for_input
         if wait_for_input is None:
             wait_for_input = not _is_in_notebook()
@@ -123,6 +137,8 @@ class FigpackView:
         if ephemeral and not upload:
             raise ValueError("ephemeral=True requires upload=True to be set")
 
+        _local_figure_name: Optional[str] = None
+
         if _dev:
             if open_in_browser:
                 print("** Note: In dev mode, open_in_browser is forced to False **")
@@ -160,7 +176,7 @@ class FigpackView:
             _local_figure_name=_local_figure_name if _dev else None,
         )
 
-    def save(self, output_path: str, *, title: str) -> None:
+    def save(self, output_path: str, *, title: str, description: str = "") -> None:
         """
         Save as figure either to a folder or to a .tar.gz file
         Args:
@@ -168,7 +184,7 @@ class FigpackView:
         """
         from ._save_figure import _save_figure
 
-        _save_figure(self, output_path, title=title)
+        _save_figure(self, output_path, title=title, description=description)
 
     def write_to_zarr_group(self, group: Group) -> None:
         """

figpack/core/zarr.py

@@ -33,13 +33,13 @@ class Group:
         if _check_zarr_version() == 2:
             self._zarr_group.create_dataset(name, **kwargs)
         elif _check_zarr_version() == 3:
-            self._zarr_group.create_array(name, **kwargs)
+            self._zarr_group.create_array(name, **kwargs)  # type: ignore
         else:
             raise RuntimeError("Unsupported Zarr version")
 
     @property
     def attrs(self) -> Dict[str, Any]:
-        return self._zarr_group.attrs
+        return self._zarr_group.attrs  # type: ignore
 
     def __getitem__(self, key: str) -> Any:
         return self._zarr_group[key]