figpack 0.2.37__py3-none-any.whl → 0.2.39__py3-none-any.whl

figpack/__init__.py

@@ -2,7 +2,7 @@
 figpack - A Python package for creating shareable, interactive visualizations in the browser
 """
 
-__version__ = "0.2.37"
+__version__ = "0.2.39"
 
 from .cli import view_figure
 from .core import FigpackView, FigpackExtension, ExtensionView

figpack/core/_bundle_utils.py

@@ -9,6 +9,7 @@ from .figpack_view import FigpackView
 from .figpack_extension import FigpackExtension
 from .extension_view import ExtensionView
 from .zarr import Group, _check_zarr_version
+from ._zarr_consolidate import consolidate_zarr_chunks
 
 thisdir = pathlib.Path(__file__).parent.resolve()
 
@@ -72,12 +73,15 @@ def prepare_figure_bundle(
         # Generate extension manifest
         _write_extension_manifest(required_extensions, tmpdir)
 
+        # Create the .zmetadata file
         zarr.consolidate_metadata(zarr_group._zarr_group.store)
 
         # It's important that we remove all the metadata files except for the
-        # consolidated one, because otherwise we may get inconstencies
-        # once we start editing the zarr data from the browser.
+        # consolidated one so there is a single source of truth.
         _remove_metadata_files_except_consolidated(pathlib.Path(tmpdir) / "data.zarr")
+
+        # Consolidate zarr chunks into larger files to reduce upload count
+        consolidate_zarr_chunks(pathlib.Path(tmpdir) / "data.zarr")
     finally:
         if _check_zarr_version() == 3:
             zarr.config.set({"default_zarr_format": old_default_zarr_format})  # type: ignore

figpack/core/_file_handler.py

@@ -42,6 +42,9 @@ class FileUploadCORSRequestHandler(CORSRequestHandler):
                 "Accept-Ranges, Content-Encoding, Content-Length, Content-Range",
             )
 
+        # Always send Accept-Ranges header to indicate byte-range support
+        self.send_header("Accept-Ranges", "bytes")
+
         # Prevent browser caching - important for when we are editing figures in place
         # This ensures the browser always fetches the latest version of files
         self.send_header("Cache-Control", "no-cache, no-store, must-revalidate")

figpack/core/_server_manager.py

@@ -29,6 +29,9 @@ class CORSRequestHandler(SimpleHTTPRequestHandler):
                 "Accept-Ranges, Content-Encoding, Content-Length, Content-Range",
             )
 
+        # Always send Accept-Ranges header to indicate byte-range support
+        self.send_header("Accept-Ranges", "bytes")
+
         # Prevent browser caching - important for when we are editing figures in place
         # This ensures the browser always fetches the latest version of files
         self.send_header("Cache-Control", "no-cache, no-store, must-revalidate")
@@ -45,6 +48,99 @@ class CORSRequestHandler(SimpleHTTPRequestHandler):
         """Reject PUT requests when file upload is not enabled."""
         self.send_error(405, "Method Not Allowed")
 
+    def do_GET(self):
+        """Handle GET requests with support for Range requests."""
+        # Translate path and check if file exists
+        path = self.translate_path(self.path)
+
+        # Check if path is a file
+        if not os.path.isfile(path):
+            # Let parent class handle directories and 404s
+            return super().do_GET()
+
+        # Check for Range header
+        range_header = self.headers.get("Range")
+
+        if range_header is None:
+            # No range request, use parent's implementation
+            return super().do_GET()
+
+        # Parse range header
+        try:
+            # Range header format: "bytes=start-end"
+            if not range_header.startswith("bytes="):
+                # Invalid range format, ignore and serve full file
+                return super().do_GET()
+
+            range_spec = range_header[6:]  # Remove "bytes=" prefix
+
+            # Get file size
+            file_size = os.path.getsize(path)
+
+            # Parse range specification
+            if "-" not in range_spec:
+                # Invalid format
+                self.send_error(400, "Invalid Range header")
+                return
+
+            range_parts = range_spec.split("-", 1)
+
+            # Determine start and end positions
+            if range_parts[0]:  # Start position specified
+                start = int(range_parts[0])
+                if range_parts[1]:  # End position also specified
+                    end = int(range_parts[1])
+                else:  # Open-ended range (e.g., "1024-")
+                    end = file_size - 1
+            else:  # Suffix range (e.g., "-500" means last 500 bytes)
+                if not range_parts[1]:
+                    self.send_error(400, "Invalid Range header")
+                    return
+                suffix_length = int(range_parts[1])
+                start = max(0, file_size - suffix_length)
+                end = file_size - 1
+
+            # Validate range
+            if start < 0 or end >= file_size or start > end:
+                self.send_response(416, "Range Not Satisfiable")
+                self.send_header("Content-Range", f"bytes */{file_size}")
+                self.end_headers()
+                return
+
+            # Calculate content length
+            content_length = end - start + 1
+
+            # Guess content type
+            import mimetypes
+
+            content_type = mimetypes.guess_type(path)[0] or "application/octet-stream"
+
+            # Send 206 Partial Content response
+            self.send_response(206, "Partial Content")
+            self.send_header("Content-Type", content_type)
+            self.send_header("Content-Length", str(content_length))
+            self.send_header("Content-Range", f"bytes {start}-{end}/{file_size}")
+            self.end_headers()
+
+            # Send the requested byte range
+            with open(path, "rb") as f:
+                f.seek(start)
+                remaining = content_length
+                while remaining > 0:
+                    chunk_size = min(8192, remaining)
+                    chunk = f.read(chunk_size)
+                    if not chunk:
+                        break
+                    self.wfile.write(chunk)
+                    remaining -= len(chunk)
+
+        except ValueError:
+            # Invalid range values
+            self.send_error(400, "Invalid Range header")
+        except Exception as e:
+            # Log error and return 500
+            self.send_error(500, f"Internal Server Error: {str(e)}")
+
     def log_message(self, format, *args):
         pass
 

figpack/core/_zarr_consolidate.py

@@ -0,0 +1,185 @@
+import os
+import pathlib
+import json
+from typing import Dict, List, Tuple
+
+
+def consolidate_zarr_chunks(
+    zarr_dir: pathlib.Path, max_file_size: int = 100_000_000
+) -> None:
+    """
+    Consolidate zarr chunk files into larger files to reduce the number of files
+    that need to be uploaded. Updates the .zmetadata file with refs mapping.
+
+    Args:
+        zarr_dir: Path to the zarr directory
+        max_file_size: Maximum size for each consolidated file in bytes (default: 100 MB)
+    """
+    if not zarr_dir.is_dir():
+        raise ValueError(f"Expected a directory, got: {zarr_dir}")
+
+    # Read the existing .zmetadata file
+    zmetadata_path = zarr_dir / ".zmetadata"
+    if not zmetadata_path.exists():
+        raise ValueError(f"No .zmetadata file found at {zmetadata_path}")
+
+    with open(zmetadata_path, "r") as f:
+        zmetadata = json.load(f)
+
+    # Collect all chunk files (non-metadata files)
+    chunk_files = _collect_chunk_files(zarr_dir)
+
+    if not chunk_files:
+        # No chunk files to consolidate
+        return
+
+    # Group chunk files into consolidated files
+    consolidated_groups = _group_files_by_size(chunk_files, max_file_size)
+
+    # Create consolidated files and build refs mapping
+    refs: Dict[str, List] = {}
+    for group_idx, file_group in enumerate(consolidated_groups):
+        consolidated_filename = f"_consolidated_{group_idx}.dat"
+        consolidated_path = zarr_dir / consolidated_filename
+
+        # Write the consolidated file and track byte offsets
+        current_offset = 0
+        with open(consolidated_path, "wb") as consolidated_file:
+            for file_path, relative_path in file_group:
+                # Read the chunk file
+                with open(file_path, "rb") as chunk_file:
+                    chunk_data = chunk_file.read()
+
+                # Write to consolidated file
+                consolidated_file.write(chunk_data)
+
+                # Add to refs mapping
+                refs[relative_path] = [
+                    consolidated_filename,
+                    current_offset,
+                    len(chunk_data),
+                ]
+
+                # Update offset
+                current_offset += len(chunk_data)
+
+    # Update .zmetadata with refs
+    zmetadata["refs"] = refs
+
+    # Write updated .zmetadata
+    with open(zmetadata_path, "w") as f:
+        json.dump(zmetadata, f, indent=2)
+
+    # Delete original chunk files
+    for file_path, _ in chunk_files:
+        try:
+            file_path.unlink()
+        except Exception as e:
+            print(f"Warning: could not remove file {file_path}: {e}")
+
+    # Clean up empty directories
+    _remove_empty_directories(zarr_dir)
+
+
+def _collect_chunk_files(zarr_dir: pathlib.Path) -> List[Tuple[pathlib.Path, str]]:
+    """
+    Collect all chunk files in the zarr directory (excluding metadata files).
+
+    Args:
+        zarr_dir: Path to the zarr directory
+
+    Returns:
+        List of tuples (absolute_path, relative_path) for each chunk file
+    """
+    chunk_files = []
+    metadata_files = {".zmetadata", ".zarray", ".zgroup", ".zattrs"}
+
+    for root, dirs, files in os.walk(zarr_dir):
+        for file in files:
+            # Skip metadata files
+            if file in metadata_files or file.startswith("_consolidated_"):
+                continue
+
+            file_path = pathlib.Path(root) / file
+            # Get relative path from zarr_dir
+            relative_path = file_path.relative_to(zarr_dir).as_posix()
+
+            chunk_files.append((file_path, relative_path))
+
+    return chunk_files
+
+
+def _group_files_by_size(
+    files: List[Tuple[pathlib.Path, str]], max_size: int
+) -> List[List[Tuple[pathlib.Path, str]]]:
+    """
+    Group files into bins where each bin's total size is <= max_size.
+
+    Uses a simple first-fit bin packing algorithm.
+
+    Args:
+        files: List of (file_path, relative_path) tuples
+        max_size: Maximum total size for each group in bytes
+
+    Returns:
+        List of groups, where each group is a list of (file_path, relative_path) tuples
+    """
+    # Get file sizes
+    files_with_sizes = []
+    for file_path, relative_path in files:
+        try:
+            size = file_path.stat().st_size
+            files_with_sizes.append((file_path, relative_path, size))
+        except Exception as e:
+            print(f"Warning: could not get size of {file_path}: {e}")
+            continue
+
+    # Sort by size (largest first) for better packing
+    files_with_sizes.sort(key=lambda x: x[2], reverse=True)
+
+    # First-fit bin packing
+    groups: List[List[Tuple[pathlib.Path, str]]] = []
+    group_sizes: List[int] = []
+
+    for file_path, relative_path, size in files_with_sizes:
+        # If file is larger than max_size, put it in its own group
+        if size > max_size:
+            groups.append([(file_path, relative_path)])
+            group_sizes.append(size)
+            continue
+
+        # Try to fit into existing group
+        placed = False
+        for i, group_size in enumerate(group_sizes):
+            if group_size + size <= max_size:
+                groups[i].append((file_path, relative_path))
+                group_sizes[i] += size
+                placed = True
+                break
+
+        # If doesn't fit anywhere, create new group
+        if not placed:
+            groups.append([(file_path, relative_path)])
+            group_sizes.append(size)
+
+    return groups
+
+
+def _remove_empty_directories(zarr_dir: pathlib.Path) -> None:
+    """
+    Remove empty directories within the zarr directory.
+
+    Args:
+        zarr_dir: Path to the zarr directory
+    """
+    # Walk bottom-up so we can remove empty parent directories
+    for root, dirs, files in os.walk(zarr_dir, topdown=False):
+        for dir_name in dirs:
+            dir_path = pathlib.Path(root) / dir_name
+            try:
+                # Only remove if directory is empty
+                if not any(dir_path.iterdir()):
+                    dir_path.rmdir()
+            except Exception:
+                # Directory not empty or other error, skip
+                pass