figpack 0.1.4__py3-none-any.whl → 0.1.6__py3-none-any.whl

figpack/__init__.py

@@ -2,8 +2,4 @@
 figpack - A Python package for creating shareable, interactive visualizations in the browser
 """
 
-__version__ = "0.1.4"
-__author__ = "Jeremy Magland"
-__email__ = "jmagland@flatironinstitute.org"
-
-from . import spike_sorting
+__version__ = "0.1.6"

figpack/cli.py

@@ -3,19 +3,20 @@ Command-line interface for figpack
 """
 
 import argparse
-import sys
+import json
 import pathlib
-import tempfile
+import socket
+import sys
 import tarfile
-import json
-import requests
+import tempfile
 import threading
 import webbrowser
-import socket
 from concurrent.futures import ThreadPoolExecutor, as_completed
-from urllib.parse import urlparse, urljoin
-from typing import Dict, List, Tuple, Optional, Union
 from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
+from typing import Dict, List, Optional, Tuple, Union
+from urllib.parse import urljoin, urlparse
+
+import requests
 
 from . import __version__
 from .core._show_view import serve_files

figpack/core/_bundle_utils.py

@@ -1,6 +1,8 @@
 import os
 import pathlib
+
 import zarr
+
 from .figpack_view import FigpackView
 
 thisdir = pathlib.Path(__file__).parent.resolve()

figpack/core/_show_view.py

@@ -1,18 +1,14 @@
 import os
-
-from typing import Union
-import tempfile
-
-import webbrowser
-
 import pathlib
-
+import tempfile
 import threading
+import webbrowser
 from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
+from typing import Union
 
-from .figpack_view import FigpackView
 from ._bundle_utils import prepare_figure_bundle
 from ._upload_bundle import _upload_bundle
+from .figpack_view import FigpackView
 
 thisdir = pathlib.Path(__file__).parent.resolve()
 
@@ -32,15 +28,15 @@ def _show_view(
 
         if upload:
             # Check for required environment variable
-            passcode = os.environ.get("FIGPACK_UPLOAD_PASSCODE")
-            if not passcode:
+            api_key = os.environ.get("FIGPACK_API_KEY")
+            if not api_key:
                 raise EnvironmentError(
-                    "FIGPACK_UPLOAD_PASSCODE environment variable must be set to upload views."
+                    "FIGPACK_API_KEY environment variable must be set to upload views."
                 )
 
             # Upload the bundle
             print("Starting upload...")
-            figure_url = _upload_bundle(tmpdir, passcode)
+            figure_url = _upload_bundle(tmpdir, api_key)
 
             if open_in_browser:
                 webbrowser.open(figure_url)

figpack/core/_upload_bundle.py

@@ -1,37 +1,71 @@
-import time
+import hashlib
 import json
-import uuid
 import pathlib
-import requests
 import threading
-import hashlib
-from .. import __version__
+import time
+import uuid
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from datetime import datetime, timedelta, timezone
 
-thisdir = pathlib.Path(__file__).parent.resolve()
+import requests
+
+from .. import __version__
 
-FIGPACK_API_BASE_URL = "https://figpack-api.vercel.app"
-TEMPORY_BASE_URL = "https://tempory.net/figpack/default/figures"
+from .config import FIGPACK_API_BASE_URL
+
+thisdir = pathlib.Path(__file__).parent.resolve()
 
 
 def _upload_single_file(
-    figure_id: str, relative_path: str, file_path: pathlib.Path, passcode: str
+    figure_url: str, relative_path: str, file_path: pathlib.Path, api_key: str
 ) -> str:
     """
-    Worker function to upload a single file
+    Worker function to upload a single file using signed URL
 
     Returns:
         str: The relative path of the uploaded file
     """
-    file_type = _determine_file_type(relative_path)
+    file_size = file_path.stat().st_size
+
+    # Get signed URL
+    payload = {
+        "figureUrl": figure_url,
+        "relativePath": relative_path,
+        "apiKey": api_key,
+        "size": file_size,
+    }
+
+    response = requests.post(f"{FIGPACK_API_BASE_URL}/api/upload", json=payload)
+
+    if not response.ok:
+        try:
+            error_data = response.json()
+            error_msg = error_data.get("message", "Unknown error")
+        except:
+            error_msg = f"HTTP {response.status_code}"
+        raise Exception(f"Failed to get signed URL for {relative_path}: {error_msg}")
 
-    if file_type == "small":
-        with open(file_path, "r", encoding="utf-8") as f:
-            content = f.read()
-        _upload_small_file(figure_id, relative_path, content, passcode)
-    else:  # large file
-        _upload_large_file(figure_id, relative_path, file_path, passcode)
+    response_data = response.json()
+    if not response_data.get("success"):
+        raise Exception(
+            f"Failed to get signed URL for {relative_path}: {response_data.get('message', 'Unknown error')}"
+        )
+
+    signed_url = response_data.get("signedUrl")
+    if not signed_url:
+        raise Exception(f"No signed URL returned for {relative_path}")
+
+    # Upload file to signed URL
+    content_type = _determine_content_type(relative_path)
+    with open(file_path, "rb") as f:
+        upload_response = requests.put(
+            signed_url, data=f, headers={"Content-Type": content_type}
+        )
+
+    if not upload_response.ok:
+        raise Exception(
+            f"Failed to upload {relative_path} to signed URL: HTTP {upload_response.status_code}"
+        )
 
     return relative_path
 
@@ -39,7 +73,7 @@ def _upload_single_file(
 MAX_WORKERS_FOR_UPLOAD = 16
 
 
-def _compute_deterministic_figure_id(tmpdir_path: pathlib.Path) -> str:
+def _compute_deterministic_figure_hash(tmpdir_path: pathlib.Path) -> str:
     """
     Compute a deterministic figure ID based on SHA1 hashes of all files
 
@@ -71,101 +105,88 @@ def _compute_deterministic_figure_id(tmpdir_path: pathlib.Path) -> str:
     return combined_hash.hexdigest()
 
 
-def _check_existing_figure(figure_id: str) -> dict:
+def _create_or_get_figure(
+    figure_hash: str, api_key: str, total_files: int = None, total_size: int = None
+) -> dict:
     """
-    Check if a figure already exists and return its status
+    Create a new figure or get existing figure information
 
     Returns:
-        dict: Contains 'exists' (bool) and 'status' (str) if exists
+        dict: Figure information from the API
     """
-    figpack_url = f"{TEMPORY_BASE_URL}/{figure_id}/figpack.json"
+    payload = {
+        "figureHash": figure_hash,
+        "apiKey": api_key,
+        "figpackVersion": __version__,
+    }
 
-    try:
-        response = requests.get(figpack_url, timeout=10)
-        if response.ok:
-            figpack_data = response.json()
-            return {"exists": True, "status": figpack_data.get("status", "unknown")}
-        else:
-            return {"exists": False}
-    except Exception:
-        return {"exists": False}
+    if total_files is not None:
+        payload["totalFiles"] = total_files
+    if total_size is not None:
+        payload["totalSize"] = total_size
 
+    response = requests.post(f"{FIGPACK_API_BASE_URL}/api/figures/create", json=payload)
 
-def _find_available_figure_id(base_figure_id: str) -> tuple:
+    if not response.ok:
+        try:
+            error_data = response.json()
+            error_msg = error_data.get("message", "Unknown error")
+        except:
+            error_msg = f"HTTP {response.status_code}"
+        raise Exception(f"Failed to create figure {figure_hash}: {error_msg}")
+
+    response_data = response.json()
+    if not response_data.get("success"):
+        raise Exception(
+            f"Failed to create figure {figure_hash}: {response_data.get('message', 'Unknown error')}"
+        )
+
+    return response_data
+
+
+def _finalize_figure(figure_url: str, api_key: str) -> dict:
     """
-    Find an available figure ID by checking base_figure_id, then base_figure_id-1, base_figure_id-2, etc.
+    Finalize a figure upload
 
     Returns:
-        tuple: (figure_id_to_use, completed_figure_id) where:
-               - figure_id_to_use is None if upload should be skipped
-               - completed_figure_id is the ID of the completed figure if one exists
+        dict: Figure information from the API
     """
-    # First check the base figure ID
-    result = _check_existing_figure(base_figure_id)
-    if not result["exists"]:
-        return (base_figure_id, None)
-    elif result["status"] == "completed":
-        print(
-            f"Figure {base_figure_id} already exists and is completed. Skipping upload."
+    payload = {
+        "figureUrl": figure_url,
+        "apiKey": api_key,
+    }
+
+    response = requests.post(
+        f"{FIGPACK_API_BASE_URL}/api/figures/finalize", json=payload
+    )
+
+    if not response.ok:
+        try:
+            error_data = response.json()
+            error_msg = error_data.get("message", "Unknown error")
+        except:
+            error_msg = f"HTTP {response.status_code}"
+        raise Exception(f"Failed to finalize figure {figure_url}: {error_msg}")
+
+    response_data = response.json()
+    if not response_data.get("success"):
+        raise Exception(
+            f"Failed to finalize figure {figure_url}: {response_data.get('message', 'Unknown error')}"
         )
-        return (None, base_figure_id)  # Signal to skip upload, return completed ID
-
-    # If exists but not completed, try with suffixes
-    suffix = 1
-    while True:
-        candidate_id = f"{base_figure_id}-{suffix}"
-        result = _check_existing_figure(candidate_id)
-
-        if not result["exists"]:
-            print(f"Using figure ID: {candidate_id}")
-            return (candidate_id, None)
-        elif result["status"] == "completed":
-            print(
-                f"Figure {candidate_id} already exists and is completed. Skipping upload."
-            )
-            return (None, candidate_id)  # Signal to skip upload, return completed ID
-
-        suffix += 1
-        if suffix > 100:  # Safety limit
-            raise Exception(
-                "Too many existing figure variants, unable to find available ID"
-            )
-
-
-def _upload_bundle(tmpdir: str, passcode: str) -> None:
+
+    return response_data
+
+
+def _upload_bundle(tmpdir: str, api_key: str) -> str:
     """
-    Upload the prepared bundle to the cloud using parallel uploads
+    Upload the prepared bundle to the cloud using the new database-driven approach
     """
     tmpdir_path = pathlib.Path(tmpdir)
 
     # Compute deterministic figure ID based on file contents
     print("Computing deterministic figure ID...")
-    base_figure_id = _compute_deterministic_figure_id(tmpdir_path)
-    print(f"Base figure ID: {base_figure_id}")
-
-    # Find available figure ID (check for existing uploads)
-    figure_id, completed_figure_id = _find_available_figure_id(base_figure_id)
-
-    # If figure_id is None, it means we found a completed upload and should skip
-    if figure_id is None:
-        figure_url = f"{TEMPORY_BASE_URL}/{completed_figure_id}/index.html"
-        print(f"Figure already exists at: {figure_url}")
-        return figure_url
-
-    print(f"Using figure ID: {figure_id}")
-
-    # First, upload initial figpack.json with "uploading" status
-    print("Uploading initial status...")
-    figpack_json = {
-        "status": "uploading",
-        "upload_started": datetime.now(timezone.utc).isoformat(),
-        "upload_updated": datetime.now(timezone.utc).isoformat(),
-        "figure_id": figure_id,
-        "figpack_version": __version__,
-    }
-    _upload_small_file(
-        figure_id, "figpack.json", json.dumps(figpack_json, indent=2), passcode
-    )
+    figure_hash = _compute_deterministic_figure_hash(tmpdir_path)
+    print(f"Figure hash: {figure_hash}")
 
     # Collect all files to upload
     all_files = []
@@ -174,34 +195,44 @@ def _upload_bundle(tmpdir: str, passcode: str) -> None:
             relative_path = file_path.relative_to(tmpdir_path)
             all_files.append((str(relative_path), file_path))
 
-    print(f"Found {len(all_files)} files to upload")
+    # Calculate total files and size for metadata
+    total_files = len(all_files)
+    total_size = sum(file_path.stat().st_size for _, file_path in all_files)
+    print(
+        f"Found {total_files} files to upload, total size: {total_size / (1024 * 1024):.2f} MB"
+    )
+
+    # Find available figure ID and create/get figure in database with metadata
+    result = _create_or_get_figure(figure_hash, api_key, total_files, total_size)
+    figure_info = result.get("figure", {})
+    figure_url = figure_info.get("figureUrl")
 
-    # Filter out figpack.json since we already uploaded the initial version
-    files_to_upload = [
-        (rel_path, file_path)
-        for rel_path, file_path in all_files
-        if rel_path != "figpack.json"
-    ]
+    if figure_info["status"] == "completed":
+        print(f"Figure already exists at: {figure_url}")
+        return figure_url
+
+    print(f"Using figure URL: {figure_url}")
+
+    files_to_upload = all_files
     total_files_to_upload = len(files_to_upload)
 
     if total_files_to_upload == 0:
-        print("No additional files to upload")
+        print("No files to upload")
     else:
         print(
-            f"Uploading {total_files_to_upload} files with up to 8 concurrent uploads..."
+            f"Uploading {total_files_to_upload} files with up to {MAX_WORKERS_FOR_UPLOAD} concurrent uploads..."
         )
 
         # Thread-safe progress tracking
         uploaded_count = 0
         count_lock = threading.Lock()
-        timer = time.time()
 
         # Upload files in parallel with concurrent uploads
         with ThreadPoolExecutor(max_workers=MAX_WORKERS_FOR_UPLOAD) as executor:
             # Submit all upload tasks
             future_to_file = {
                 executor.submit(
-                    _upload_single_file, figure_id, rel_path, file_path, passcode
+                    _upload_single_file, figure_url, rel_path, file_path, api_key
                 ): rel_path
                 for rel_path, file_path in files_to_upload
             }
@@ -219,36 +250,14 @@ def _upload_bundle(tmpdir: str, passcode: str) -> None:
                             f"Uploaded {uploaded_count}/{total_files_to_upload}: {relative_path}"
                         )
 
-                        # Update progress every 60 seconds
-                        elapsed_time = time.time() - timer
-                        if elapsed_time > 60:
-                            figpack_json = {
-                                **figpack_json,
-                                "status": "uploading",
-                                "upload_progress": f"{uploaded_count}/{total_files_to_upload}",
-                                "upload_updated": datetime.now(
-                                    timezone.utc
-                                ).isoformat(),
-                            }
-                            _upload_small_file(
-                                figure_id,
-                                "figpack.json",
-                                json.dumps(figpack_json, indent=2),
-                                passcode,
-                            )
-                            print(
-                                f"Updated figpack.json with progress: {uploaded_count}/{total_files_to_upload}"
-                            )
-                            timer = time.time()
-
                 except Exception as e:
                     print(f"Failed to upload {relative_path}: {e}")
                     raise  # Re-raise the exception to stop the upload process
 
-    # Create and upload manifest.json
-    print("Creating manifest.json...")
+    # Create manifest for finalization
+    print("Creating manifest...")
     manifest = {
-        "timestamp": datetime.now(timezone.utc).isoformat(),
+        "timestamp": time.time(),
         "files": [],
         "total_size": 0,
         "total_files": len(files_to_upload),
@@ -259,175 +268,58 @@ def _upload_bundle(tmpdir: str, passcode: str) -> None:
         manifest["files"].append({"path": rel_path, "size": file_size})
         manifest["total_size"] += file_size
 
-    _upload_small_file(
-        figure_id, "manifest.json", json.dumps(manifest, indent=2), passcode
-    )
-    print("Uploaded manifest.json")
     print(f"Total size: {manifest['total_size'] / (1024 * 1024):.2f} MB")
 
-    # Finally, upload completion status
-    figpack_json = {
-        **figpack_json,
-        "status": "completed",
-        "upload_completed": datetime.now(timezone.utc).isoformat(),
-        "expiration": (datetime.now(timezone.utc) + timedelta(days=1)).isoformat(),
-        "figure_id": figure_id,
-        "total_files": len(all_files),
-        "total_size": manifest["total_size"],
-        "figpack_version": __version__,
-    }
-    _upload_small_file(
-        figure_id, "figpack.json", json.dumps(figpack_json, indent=2), passcode
-    )
-    print("Upload completed successfully")
-
-    figure_url = f"{TEMPORY_BASE_URL}/{figure_id}/index.html"
-    return figure_url
-
-
-def _determine_file_type(file_path: str) -> str:
-    """
-    Determine if a file should be uploaded as small or large
-    Based on the validation logic in the API
-    """
-    # Check exact matches first
-    if file_path == "figpack.json" or file_path == "index.html":
-        return "small"
-
-    # Check zarr metadata files
-    if (
-        file_path.endswith(".zattrs")
-        or file_path.endswith(".zgroup")
-        or file_path.endswith(".zarray")
-        or file_path.endswith(".zmetadata")
-    ):
-        return "small"
-
-    # Check HTML files
-    if file_path.endswith(".html"):
-        return "small"
-
-    # Check data.zarr directory
-    if file_path.startswith("data.zarr/"):
-        file_name = file_path[len("data.zarr/") :]
-        # Check if it's a zarr chunk (numeric like 0.0.1)
-        if _is_zarr_chunk(file_name):
-            return "large"
-        # Check for zarr metadata files in subdirectories
-        if (
-            file_name.endswith(".zattrs")
-            or file_name.endswith(".zgroup")
-            or file_name.endswith(".zarray")
-            or file_name.endswith(".zmetadata")
-        ):
-            return "small"
-
-    # Check assets directory
-    if file_path.startswith("assets/"):
-        file_name = file_path[len("assets/") :]
-        if file_name.endswith(".js") or file_name.endswith(".css"):
-            return "large"
-
-    # Default to large file
-    return "large"
-
-
-def _is_zarr_chunk(file_name: str) -> bool:
-    """
-    Check if filename consists only of numbers and dots (zarr chunk pattern)
-    """
-    for char in file_name:
-        if char != "." and not char.isdigit():
-            return False
-    return (
-        len(file_name) > 0
-        and not file_name.startswith(".")
-        and not file_name.endswith(".")
-    )
+    # Upload manifest.json
+    print("Uploading manifest.json...")
+    manifest_content = json.dumps(manifest, indent=2)
+    manifest_size = len(manifest_content.encode("utf-8"))
 
-
-def _upload_small_file(
-    figure_id: str, file_path: str, content: str, passcode: str
-) -> None:
-    """
-    Upload a small file by sending content directly
-    """
-    destination_url = f"{TEMPORY_BASE_URL}/{figure_id}/{file_path}"
-
-    try:
-        content.encode("utf-8")
-    except Exception as e:
-        raise Exception(f"Content for {file_path} is not UTF-8 encodable: {e}")
-    payload = {
-        "destinationUrl": destination_url,
-        "passcode": passcode,
-        "content": content,
+    manifest_payload = {
+        "figureUrl": figure_url,
+        "relativePath": "manifest.json",
+        "apiKey": api_key,
+        "size": manifest_size,
     }
-    # check that payload is json serializable
-    try:
-        json.dumps(payload)
-    except Exception as e:
-        raise Exception(f"Payload for {file_path} is not JSON serializable: {e}")
-
-    response = requests.post(f"{FIGPACK_API_BASE_URL}/api/upload", json=payload)
-
-    if not response.ok:
-        try:
-            error_data = response.json()
-            error_msg = error_data.get("message", "Unknown error")
-        except:
-            error_msg = f"HTTP {response.status_code}"
-        raise Exception(f"Failed to upload {file_path}: {error_msg}")
-
-
-def _upload_large_file(
-    figure_id: str, file_path: str, local_file_path: pathlib.Path, passcode: str
-) -> None:
-    """
-    Upload a large file using signed URL
-    """
-    destination_url = f"{TEMPORY_BASE_URL}/{figure_id}/{file_path}"
-    file_size = local_file_path.stat().st_size
-
-    # Get signed URL
-    payload = {
-        "destinationUrl": destination_url,
-        "passcode": passcode,
-        "size": file_size,
-    }
-
-    response = requests.post(f"{FIGPACK_API_BASE_URL}/api/upload", json=payload)
 
+    response = requests.post(
+        f"{FIGPACK_API_BASE_URL}/api/upload", json=manifest_payload
+    )
     if not response.ok:
         try:
             error_data = response.json()
             error_msg = error_data.get("message", "Unknown error")
         except:
             error_msg = f"HTTP {response.status_code}"
-        raise Exception(f"Failed to get signed URL for {file_path}: {error_msg}")
+        raise Exception(f"Failed to get signed URL for manifest.json: {error_msg}")
 
     response_data = response.json()
     if not response_data.get("success"):
         raise Exception(
-            f"Failed to get signed URL for {file_path}: {response_data.get('message', 'Unknown error')}"
+            f"Failed to get signed URL for manifest.json: {response_data.get('message', 'Unknown error')}"
         )
 
     signed_url = response_data.get("signedUrl")
     if not signed_url:
-        raise Exception(f"No signed URL returned for {file_path}")
+        raise Exception("No signed URL returned for manifest.json")
 
-    # Upload file to signed URL
-    content_type = _determine_content_type(file_path)
-    with open(local_file_path, "rb") as f:
-        upload_response = requests.put(
-            signed_url, data=f, headers={"Content-Type": content_type}
-        )
+    # Upload manifest using signed URL
+    upload_response = requests.put(
+        signed_url, data=manifest_content, headers={"Content-Type": "application/json"}
+    )
 
     if not upload_response.ok:
         raise Exception(
-            f"Failed to upload {file_path} to signed URL: HTTP {upload_response.status_code}"
+            f"Failed to upload manifest.json to signed URL: HTTP {upload_response.status_code}"
         )
 
+    # Finalize the figure upload
+    print("Finalizing figure...")
+    _finalize_figure(figure_url, api_key)
+    print("Upload completed successfully")
+
+    return figure_url
+
 
 def _determine_content_type(file_path: str) -> str:
     """

figpack/core/config.py

@@ -0,0 +1 @@
+FIGPACK_API_BASE_URL = "https://figpack-api.vercel.app"

figpack/core/figpack_view.py

@@ -49,6 +49,7 @@ class FigpackView:
             print(
                 f"For development, run figpack-gui in dev mode and use http://localhost:5173?data=http://localhost:{port}/data.zarr"
             )
+            open_in_browser = False
 
         _show_view(
             self,