figpack 0.2.17__py3-none-any.whl → 0.2.40__py3-none-any.whl

figpack/__init__.py

@@ -2,7 +2,7 @@
 figpack - A Python package for creating shareable, interactive visualizations in the browser
 """
 
-__version__ = "0.2.17"
+__version__ = "0.2.40"
 
 from .cli import view_figure
 from .core import FigpackView, FigpackExtension, ExtensionView

figpack/cli.py

@@ -18,6 +18,8 @@ import requests
 from . import __version__
 from .core._server_manager import CORSRequestHandler
 from .core._view_figure import serve_files, view_figure
+from .core._upload_bundle import _upload_bundle, get_figure_by_source_url
+from .extensions import ExtensionManager
 
 MAX_WORKERS_FOR_DOWNLOAD = 16
 
@@ -214,6 +216,222 @@ def download_figure(figure_url: str, dest_path: str) -> None:
         print(f"Archive saved to: {dest_path}")
 
 
+def handle_extensions_command(args):
+    """Handle extensions subcommands"""
+    extension_manager = ExtensionManager()
+
+    if args.extensions_command == "list":
+        extension_manager.list_extensions()
+    elif args.extensions_command == "install":
+        if not args.extensions and not args.all:
+            print("Error: No extensions specified. Use extension names or --all flag.")
+            print("Example: figpack extensions install figpack_3d")
+            print("         figpack extensions install --all")
+            sys.exit(1)
+
+        success = extension_manager.install_extensions(
+            extensions=args.extensions, upgrade=args.upgrade, install_all=args.all
+        )
+
+        if not success:
+            sys.exit(1)
+
+    elif args.extensions_command == "uninstall":
+        success = extension_manager.uninstall_extensions(args.extensions)
+
+        if not success:
+            sys.exit(1)
+    else:
+        print("Available extension commands:")
+        print("  list      - List available extensions and their status")
+        print("  install   - Install or upgrade extension packages")
+        print("  uninstall - Uninstall extension packages")
+        print()
+        print("Use 'figpack extensions <command> --help' for more information.")
+
+
+def handle_upload_from_source_url(args) -> None:
+    """
+    Handle the upload-from-source-url command
+
+    Downloads a tar.gz/tgz file from a URL, extracts it, and uploads it as a new figure
+    with the source URL set.
+    """
+    import os
+
+    source_url = args.source_url
+    title = args.title if hasattr(args, "title") else None
+
+    # Get API key from environment variable
+    api_key = os.environ.get("FIGPACK_API_KEY")
+    if not api_key:
+        print(
+            "Error: FIGPACK_API_KEY environment variable must be set to upload figures."
+        )
+        sys.exit(1)
+
+    # Validate URL format
+    if not (source_url.endswith(".tar.gz") or source_url.endswith(".tgz")):
+        print(f"Error: Source URL must point to a .tar.gz or .tgz file: {source_url}")
+        sys.exit(1)
+
+    print(f"Downloading archive from: {source_url}")
+
+    try:
+        # Download the archive
+        response = requests.get(source_url, timeout=120, stream=True)
+        response.raise_for_status()
+
+        # Create temporary file for the archive
+        with tempfile.NamedTemporaryFile(
+            suffix=".tar.gz", delete=False
+        ) as temp_archive:
+            archive_path = temp_archive.name
+
+            # Download with progress indication
+            total_size = int(response.headers.get("content-length", 0))
+            downloaded_size = 0
+
+            for chunk in response.iter_content(chunk_size=8192):
+                if chunk:
+                    temp_archive.write(chunk)
+                    downloaded_size += len(chunk)
+                    if total_size > 0:
+                        progress = (downloaded_size / total_size) * 100
+                        print(
+                            f"Downloaded: {downloaded_size / (1024*1024):.2f} MB ({progress:.1f}%)",
+                            end="\r",
+                        )
+
+            if total_size > 0:
+                print()  # New line after progress
+            print(f"Download complete: {downloaded_size / (1024*1024):.2f} MB")
+
+        # Extract archive to temporary directory
+        print("Extracting archive...")
+        with tempfile.TemporaryDirectory() as extract_dir:
+            extract_path = pathlib.Path(extract_dir)
+
+            try:
+                with tarfile.open(archive_path, "r:gz") as tar:
+                    tar.extractall(path=extract_path)
+                print(f"Extracted to temporary directory")
+            except Exception as e:
+                print(f"Error: Failed to extract archive: {e}")
+                sys.exit(1)
+            finally:
+                # Clean up downloaded archive
+                import os
+
+                try:
+                    os.unlink(archive_path)
+                except Exception:
+                    pass
+
+            # Upload the extracted files
+            print(f"Uploading figure with source URL: {source_url}")
+            try:
+                figure_url = _upload_bundle(
+                    str(extract_path),
+                    api_key=api_key,
+                    title=title,
+                    source_url=source_url,
+                )
+                print(f"\nFigure uploaded successfully!")
+                print(f"Figure URL: {figure_url}")
+            except Exception as e:
+                print(f"Error: Failed to upload figure: {e}")
+                sys.exit(1)
+
+    except requests.exceptions.RequestException as e:
+        print(f"Error: Failed to download archive from {source_url}: {e}")
+        sys.exit(1)
+    except Exception as e:
+        print(f"Error: {e}")
+        sys.exit(1)
+
+
+def handle_find_by_source_url(args) -> None:
+    """
+    Handle the find-by-source-url command
+
+    Queries the API for a figure URL by its source URL.
+    """
+    source_url = args.source_url
+
+    print(f"Querying for figure with source URL: {source_url}")
+
+    try:
+        figure_url = get_figure_by_source_url(source_url)
+
+        if figure_url:
+            print(f"Figure found: {figure_url}")
+        else:
+            print(f"No figure found with source URL: {source_url}")
+            sys.exit(1)
+    except Exception as e:
+        print(f"Error: {e}")
+        sys.exit(1)
+
+
+def download_and_view_archive(url: str, port: int = None) -> None:
+    """
+    Download a tar.gz/tgz archive from a URL and view it
+
+    Args:
+        url: URL to the tar.gz or tgz file
+        port: Optional port number to serve on
+    """
+    if not (url.endswith(".tar.gz") or url.endswith(".tgz")):
+        print(f"Error: URL must point to a .tar.gz or .tgz file: {url}")
+        sys.exit(1)
+
+    print(f"Downloading archive from: {url}")
+
+    try:
+        response = requests.get(url, timeout=60, stream=True)
+        response.raise_for_status()
+
+        # Create a temporary file to store the downloaded archive
+        with tempfile.NamedTemporaryFile(suffix=".tar.gz", delete=False) as temp_file:
+            temp_path = temp_file.name
+
+            # Download with progress indication
+            total_size = int(response.headers.get("content-length", 0))
+            downloaded_size = 0
+
+            for chunk in response.iter_content(chunk_size=8192):
+                if chunk:
+                    temp_file.write(chunk)
+                    downloaded_size += len(chunk)
+                    if total_size > 0:
+                        progress = (downloaded_size / total_size) * 100
+                        print(
+                            f"Downloaded: {downloaded_size / (1024*1024):.2f} MB ({progress:.1f}%)",
+                            end="\r",
+                        )
+
+            if total_size > 0:
+                print()  # New line after progress
+            print(f"Download complete: {downloaded_size / (1024*1024):.2f} MB")
+
+        # Now view the downloaded file
+        try:
+            view_figure(temp_path, port=port)
+        finally:
+            # Clean up the temporary file after viewing
+            import os
+
+            try:
+                os.unlink(temp_path)
+            except Exception:
+                pass
+
+    except requests.exceptions.RequestException as e:
+        print(f"Error: Failed to download archive from {url}: {e}")
+        sys.exit(1)
+
+
 def main():
     """Main CLI entry point"""
     parser = argparse.ArgumentParser(
@@ -235,17 +453,85 @@ def main():
     view_parser = subparsers.add_parser(
         "view", help="Extract and serve a figure archive locally"
     )
-    view_parser.add_argument("archive", help="Path to the tar.gz archive file")
+    view_parser.add_argument("archive", help="Path or URL to the tar.gz archive file")
     view_parser.add_argument(
         "--port", type=int, help="Port number to serve on (default: auto-select)"
     )
 
+    # Extensions command
+    extensions_parser = subparsers.add_parser(
+        "extensions", help="Manage figpack extension packages"
+    )
+    extensions_subparsers = extensions_parser.add_subparsers(
+        dest="extensions_command", help="Extension management commands"
+    )
+
+    # Extensions list subcommand
+    extensions_list_parser = extensions_subparsers.add_parser(
+        "list", help="List available extensions and their status"
+    )
+
+    # Extensions install subcommand
+    extensions_install_parser = extensions_subparsers.add_parser(
+        "install", help="Install or upgrade extension packages"
+    )
+    extensions_install_parser.add_argument(
+        "extensions",
+        nargs="*",
+        help="Extension package names to install (e.g., figpack_3d figpack_spike_sorting)",
+    )
+    extensions_install_parser.add_argument(
+        "--all", action="store_true", help="Install all available extensions"
+    )
+    extensions_install_parser.add_argument(
+        "--upgrade", action="store_true", help="Upgrade packages if already installed"
+    )
+
+    # Extensions uninstall subcommand
+    extensions_uninstall_parser = extensions_subparsers.add_parser(
+        "uninstall", help="Uninstall extension packages"
+    )
+    extensions_uninstall_parser.add_argument(
+        "extensions", nargs="+", help="Extension package names to uninstall"
+    )
+
+    # Upload from URL command
+    upload_from_source_url_parser = subparsers.add_parser(
+        "upload-from-source-url",
+        help="Download a tar.gz/tgz file and upload it as a new figure",
+    )
+    upload_from_source_url_parser.add_argument(
+        "source_url",
+        help="URL to the tar.gz or tgz file (will be set as the figure's source URL)",
+    )
+    upload_from_source_url_parser.add_argument(
+        "--title", help="Optional title for the figure"
+    )
+
+    # Find by source URL command
+    find_by_source_url_parser = subparsers.add_parser(
+        "find-by-source-url", help="Get the figure URL for a given source URL"
+    )
+    find_by_source_url_parser.add_argument(
+        "source_url", help="The source URL to search for"
+    )
+
     args = parser.parse_args()
 
     if args.command == "download":
         download_figure(args.figure_url, args.dest)
     elif args.command == "view":
-        view_figure(args.archive, port=args.port)
+        # Check if archive argument is a URL
+        if args.archive.startswith("http://") or args.archive.startswith("https://"):
+            download_and_view_archive(args.archive, port=args.port)
+        else:
+            view_figure(args.archive, port=args.port)
+    elif args.command == "extensions":
+        handle_extensions_command(args)
+    elif args.command == "upload-from-source-url":
+        handle_upload_from_source_url(args)
+    elif args.command == "find-by-source-url":
+        handle_find_by_source_url(args)
     else:
         parser.print_help()
 

figpack/core/_bundle_utils.py

@@ -1,7 +1,7 @@
 import os
 import pathlib
 import json
-from typing import Set
+from typing import Optional, List
 
 import zarr
 
@@ -9,12 +9,13 @@ from .figpack_view import FigpackView
 from .figpack_extension import FigpackExtension
 from .extension_view import ExtensionView
 from .zarr import Group, _check_zarr_version
+from ._zarr_consolidate import consolidate_zarr_chunks
 
 thisdir = pathlib.Path(__file__).parent.resolve()
 
 
 def prepare_figure_bundle(
-    view: FigpackView, tmpdir: str, *, title: str, description: str = None
+    view: FigpackView, tmpdir: str, *, title: str, description: Optional[str] = None
 ) -> None:
     """
     Prepare a figure bundle in the specified temporary directory.
@@ -51,14 +52,14 @@ def prepare_figure_bundle(
     # because we only support version 2 on the frontend right now.
 
     if _check_zarr_version() == 3:
-        old_default_zarr_format = zarr.config.get("default_zarr_format")
-        zarr.config.set({"default_zarr_format": 2})
+        old_default_zarr_format = zarr.config.get("default_zarr_format")  # type: ignore
+        zarr.config.set({"default_zarr_format": 2})  # type: ignore
 
     try:
         # Write the view data to the Zarr group
         zarr_group = zarr.open_group(pathlib.Path(tmpdir) / "data.zarr", mode="w")
         zarr_group = Group(zarr_group)
-        view._write_to_zarr_group(zarr_group)
+        view.write_to_zarr_group(zarr_group)
 
         # Add title and description as attributes on the top-level zarr group
         zarr_group.attrs["title"] = title
@@ -72,13 +73,45 @@ def prepare_figure_bundle(
         # Generate extension manifest
         _write_extension_manifest(required_extensions, tmpdir)
 
+        # Create the .zmetadata file
         zarr.consolidate_metadata(zarr_group._zarr_group.store)
+
+        # It's important that we remove all the metadata files except for the
+        # consolidated one so there is a single source of truth.
+        _remove_metadata_files_except_consolidated(pathlib.Path(tmpdir) / "data.zarr")
+
+        # Consolidate zarr chunks into larger files to reduce upload count
+        consolidate_zarr_chunks(pathlib.Path(tmpdir) / "data.zarr")
     finally:
         if _check_zarr_version() == 3:
-            zarr.config.set({"default_zarr_format": old_default_zarr_format})
+            zarr.config.set({"default_zarr_format": old_default_zarr_format})  # type: ignore
+
 
+def _remove_metadata_files_except_consolidated(zarr_dir: pathlib.Path) -> None:
+    """
+    Remove all zarr metadata files except for the consolidated one.
 
-def _discover_required_extensions(view: FigpackView) -> Set[str]:
+    Args:
+        zarr_dir: Path to the zarr directory
+    """
+    if not zarr_dir.is_dir():
+        raise ValueError(f"Expected a directory, got: {zarr_dir}")
+
+    for root, dirs, files in os.walk(zarr_dir):
+        for file in files:
+            if (
+                file.endswith(".zarray")
+                or file.endswith(".zgroup")
+                or file.endswith(".zattrs")
+            ):
+                file_path = pathlib.Path(root) / file
+                try:
+                    file_path.unlink()
+                except Exception as e:
+                    print(f"Warning: could not remove file {file_path}: {e}")
+
+
+def _discover_required_extensions(view: FigpackView) -> List[str]:
     """
     Recursively discover all extensions required by a view and its children
 

figpack/core/_file_handler.py

@@ -0,0 +1,195 @@
+import os
+import pathlib
+import urllib.parse
+from http.server import SimpleHTTPRequestHandler
+from typing import Optional
+
+from ._server_manager import CORSRequestHandler
+
+
+class FileUploadCORSRequestHandler(CORSRequestHandler):
+    """
+    Extended CORS request handler that supports PUT requests for file uploads.
+    Only allows file operations within the served directory.
+    """
+
+    def __init__(
+        self,
+        *args,
+        allow_origin=None,
+        enable_file_upload=False,
+        max_file_size=10 * 1024 * 1024,
+        **kwargs,
+    ):
+        self.enable_file_upload = enable_file_upload
+        self.max_file_size = max_file_size  # Default 10MB
+        super().__init__(*args, allow_origin=allow_origin, **kwargs)
+
+    def end_headers(self):
+        if self.allow_origin is not None:
+            self.send_header("Access-Control-Allow-Origin", self.allow_origin)
+            self.send_header("Vary", "Origin")
+            # Add PUT to allowed methods if file upload is enabled
+            methods = "GET, HEAD, OPTIONS"
+            if self.enable_file_upload:
+                methods += ", PUT"
+            self.send_header("Access-Control-Allow-Methods", methods)
+            self.send_header(
+                "Access-Control-Allow-Headers", "Content-Type, Range, Content-Length"
+            )
+            self.send_header(
+                "Access-Control-Expose-Headers",
+                "Accept-Ranges, Content-Encoding, Content-Length, Content-Range",
+            )
+
+        # Always send Accept-Ranges header to indicate byte-range support
+        self.send_header("Accept-Ranges", "bytes")
+
+        # Prevent browser caching - important for when we are editing figures in place
+        # This ensures the browser always fetches the latest version of files
+        self.send_header("Cache-Control", "no-cache, no-store, must-revalidate")
+        self.send_header("Pragma", "no-cache")
+        self.send_header("Expires", "0")
+
+        super(SimpleHTTPRequestHandler, self).end_headers()
+
+    def do_PUT(self):
+        """Handle PUT requests for file uploads."""
+        if not self.enable_file_upload:
+            self.send_error(405, "Method Not Allowed")
+            return
+
+        try:
+            # Parse and validate the path
+            file_path = self._get_safe_file_path()
+            if file_path is None:
+                return  # Error already sent
+
+            # Check content length
+            content_length = self._get_content_length()
+            if content_length is None:
+                return  # Error already sent
+
+            # Determine if this will be a create or update
+            is_new_file = not file_path.exists()
+
+            # Read and write the file
+            if self._write_file_content(file_path, content_length):
+                # Send appropriate status code
+                status_code = 201 if is_new_file else 200
+                self.send_response(status_code)
+                self.send_header("Content-Type", "application/json")
+                self.end_headers()
+
+                response_data = f'{{"status": "success", "path": "{file_path.relative_to(pathlib.Path(self.directory))}"}}'
+                self.wfile.write(response_data.encode("utf-8"))
+
+        except Exception as e:
+            self.log_error(f"Error in PUT request: {e}")
+            self.send_error(500, f"Internal Server Error: {str(e)}")
+
+    def _get_safe_file_path(self) -> Optional[pathlib.Path]:
+        """
+        Parse and validate the requested file path.
+        Returns None if the path is invalid or unsafe.
+        """
+        # Parse the URL path
+        parsed_path = urllib.parse.urlparse(self.path).path
+
+        # Remove leading slash and decode URL encoding
+        relative_path = urllib.parse.unquote(parsed_path.lstrip("/"))
+
+        # Prevent empty paths
+        if not relative_path:
+            self.send_error(400, "Bad Request: Empty file path")
+            return None
+
+        # Get the served directory
+        served_dir = pathlib.Path(self.directory).resolve()
+
+        # Construct the target file path
+        target_path = served_dir / relative_path
+
+        try:
+            # Resolve the path to handle any .. or . components
+            resolved_path = target_path.resolve()
+
+            # Ensure the resolved path is within the served directory
+            if not str(resolved_path).startswith(str(served_dir)):
+                self.send_error(403, "Forbidden: Path outside served directory")
+                return None
+
+        except (OSError, ValueError) as e:
+            self.send_error(400, f"Bad Request: Invalid path - {str(e)}")
+            return None
+
+        return resolved_path
+
+    def _get_content_length(self) -> Optional[int]:
+        """
+        Get and validate the content length from headers.
+        Returns None if invalid or too large.
+        """
+        content_length_header = self.headers.get("Content-Length")
+        if not content_length_header:
+            self.send_error(400, "Bad Request: Content-Length header required")
+            return None
+
+        try:
+            content_length = int(content_length_header)
+        except ValueError:
+            self.send_error(400, "Bad Request: Invalid Content-Length")
+            return None
+
+        if content_length < 0:
+            self.send_error(400, "Bad Request: Negative Content-Length")
+            return None
+
+        if content_length > self.max_file_size:
+            self.send_error(
+                413,
+                f"Payload Too Large: Maximum file size is {self.max_file_size} bytes",
+            )
+            return None
+
+        return content_length
+
+    def _write_file_content(self, file_path: pathlib.Path, content_length: int) -> bool:
+        """
+        Write the request body content to the specified file.
+        Returns True on success, False on failure (error already sent).
+        """
+        try:
+            # Create parent directories if they don't exist
+            file_path.parent.mkdir(parents=True, exist_ok=True)
+
+            # Write the file content
+            with open(file_path, "wb") as f:
+                remaining = content_length
+                while remaining > 0:
+                    # Read in chunks to handle large files efficiently
+                    chunk_size = min(8192, remaining)
+                    chunk = self.rfile.read(chunk_size)
+
+                    if not chunk:
+                        # Unexpected end of data
+                        self.send_error(400, "Bad Request: Incomplete data")
+                        return False
+
+                    f.write(chunk)
+                    remaining -= len(chunk)
+
+            return True
+
+        except OSError as e:
+            self.send_error(
+                500, f"Internal Server Error: Could not write file - {str(e)}"
+            )
+            return False
+        except Exception as e:
+            self.send_error(500, f"Internal Server Error: {str(e)}")
+            return False
+
+    def log_message(self, format, *args):
+        """Override to suppress default logging (same as parent class)."""
+        pass

figpack/core/_save_figure.py

@@ -5,7 +5,9 @@ from ._bundle_utils import prepare_figure_bundle
 from .figpack_view import FigpackView
 
 
-def _save_figure(view: FigpackView, output_path: str, *, title: str):
+def _save_figure(
+    view: FigpackView, output_path: str, *, title: str, description: str = ""
+) -> None:
     """
     Save the figure to a folder or a .tar.gz file
 
@@ -13,19 +15,21 @@ def _save_figure(view: FigpackView, output_path: str, *, title: str):
         view: FigpackView instance to save
         output_path: Output path (destination folder or .tar.gz file path)
     """
-    output_path = pathlib.Path(output_path)
-    if (output_path.suffix == ".gz" and output_path.suffixes[-2] == ".tar") or (
-        output_path.suffix == ".tgz"
+    output_path_2 = pathlib.Path(output_path)
+    if (output_path_2.suffix == ".gz" and output_path_2.suffixes[-2] == ".tar") or (
+        output_path_2.suffix == ".tgz"
     ):
         # It's a .tar.gz file
         with tempfile.TemporaryDirectory(prefix="figpack_save_") as tmpdir:
-            prepare_figure_bundle(view, tmpdir, title=title)
+            prepare_figure_bundle(view, tmpdir, title=title, description=description)
             # Create tar.gz file
             import tarfile
 
-            with tarfile.open(output_path, "w:gz") as tar:
+            with tarfile.open(output_path_2, "w:gz") as tar:
                 tar.add(tmpdir, arcname=".")
     else:
         # It's a folder
-        output_path.mkdir(parents=True, exist_ok=True)
-        prepare_figure_bundle(view, str(output_path), title=title)
+        output_path_2.mkdir(parents=True, exist_ok=True)
+        prepare_figure_bundle(
+            view, str(output_path_2), title=title, description=description
+        )