supernote 0.9.2__tar.gz → 0.11.0__tar.gz

{supernote-0.9.2 → supernote-0.11.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: supernote
-Version: 0.9.2
+Version: 0.11.0
 Summary: All-in-one toolkit for Supernote devices: parse notebooks, self-host services, access services
 Author-email: jya <jya@wizmy.net>, Allen Porter <allen.porter@gmail.com>
 License-Expression: Apache-2.0

{supernote-0.9.2 → supernote-0.11.0}/pyproject.toml

@@ -4,7 +4,7 @@ requires = ["setuptools>=77.0"]
 
 [project]
 name = "supernote"
-version = "0.9.2"
+version = "0.11.0"
 license = "Apache-2.0"
 license-files = ["LICENSE"]
 description = "All-in-one toolkit for Supernote devices: parse notebooks, self-host services, access services"

{supernote-0.9.2 → supernote-0.11.0}/supernote/client/device.py

@@ -180,6 +180,7 @@ class DeviceClient:
             path=parent_path_str,
             content_hash=md5,
             equipment_no=equipment_no,
+            inner_name=apply.inner_name,
         )
 
     async def upload_apply(
@@ -199,6 +200,7 @@ class DeviceClient:
         path: str,
         content_hash: str,
         equipment_no: str,
+        inner_name: str | None = None,
     ) -> FileUploadFinishLocalVO:
         """Finish file upload."""
         dto = FileUploadFinishLocalDTO(
@@ -206,6 +208,7 @@ class DeviceClient:
             path=path,
             content_hash=content_hash,
             equipment_no=equipment_no,
+            inner_name=inner_name,
         )
         return await self._client.post_json(
             "/api/file/2/files/upload/finish",

{supernote-0.9.2 → supernote-0.11.0}/supernote/server/app.py

@@ -209,6 +209,7 @@ def create_app(config: ServerConfig) -> web.Application:
 
     app["session_manager"] = session_manager
     app["coordination_service"] = coordination_service
+    app["blob_storage"] = blob_storage
     user_service = UserService(config.auth, coordination_service, session_manager)
     file_service = FileService(
         config.storage_root,

{supernote-0.9.2 → supernote-0.11.0}/supernote/server/config.py

@@ -16,16 +16,25 @@ class AuthConfig(DataClassYAMLMixin):
     """Authentication configuration."""
 
     secret_key: str = ""
-    """JWT secret key."""
+    """JWT secret key.
+    
+    Env Var: `SUPERNOTE_JWT_SECRET`
+    """
 
     expiration_hours: int = 24
     """JWT expiration time in hours."""
 
     enable_registration: bool = False
-    """When disabled, registration is only allowed if there are no users in the system."""
+    """When disabled, registration is only allowed if there are no users in the system.
+    
+    Env Var: `SUPERNOTE_ENABLE_REGISTRATION`
+    """
 
     enable_remote_password_reset: bool = False
-    """When disabled, the public password reset endpoint returns 403."""
+    """When disabled, the public password reset endpoint returns 403.
+    
+    Env Var: `SUPERNOTE_ENABLE_REMOTE_PASSWORD_RESET`
+    """
 
     class Config(BaseConfig):
         omit_none = True
@@ -35,17 +44,36 @@ class AuthConfig(DataClassYAMLMixin):
 @dataclass
 class ServerConfig(DataClassYAMLMixin):
     host: str = "0.0.0.0"
+    """Host to bind the server to.
+    
+    Env Var: `SUPERNOTE_HOST`
+    """
+
     port: int = 8080
+    """Port to bind the server to.
+    
+    Env Var: `SUPERNOTE_PORT`
+    """
     trace_log_file: str | None = None
     storage_dir: str = "storage"
+    """Directory for storing files and database.
+    
+    Env Var: `SUPERNOTE_STORAGE_DIR`
+    """
 
     proxy_mode: str | None = None
-    """Proxy header handling mode: None/'disabled' (ignore proxy headers), 'relaxed' (trust immediate upstream), or 'strict' (require specific trusted IPs). Defaults to None for security."""
+    """Proxy header handling mode: None/'disabled' (ignore proxy headers), 'relaxed' (trust immediate upstream), or 'strict' (require specific trusted IPs). Defaults to None for security.
+    
+    Env Var: `SUPERNOTE_PROXY_MODE`
+    """
 
     trusted_proxies: list[str] = field(
         default_factory=lambda: ["127.0.0.1", "::1", "172.17.0.0/16"]
     )
-    """List of trusted proxy IPs/networks (used in strict mode). Supports CIDR notation."""
+    """List of trusted proxy IPs/networks (used in strict mode). Supports CIDR notation.
+    
+    Env Var: `SUPERNOTE_TRUSTED_PROXIES` (comma-separated)
+    """
 
     auth: AuthConfig = field(default_factory=AuthConfig)
 
@@ -120,6 +148,15 @@ class ServerConfig(DataClassYAMLMixin):
                 f"Remote Password Reset Enabled: {config.auth.enable_remote_password_reset}"
             )
 
+        if os.getenv("SUPERNOTE_PROXY_MODE"):
+            config.proxy_mode = os.getenv("SUPERNOTE_PROXY_MODE")
+            logger.info(f"Using SUPERNOTE_PROXY_MODE: {config.proxy_mode}")
+
+        if os.getenv("SUPERNOTE_TRUSTED_PROXIES"):
+            val = os.getenv("SUPERNOTE_TRUSTED_PROXIES", "")
+            config.trusted_proxies = [p.strip() for p in val.split(",") if p.strip()]
+            logger.info(f"Using SUPERNOTE_TRUSTED_PROXIES: {config.trusted_proxies}")
+
         if not config_file.exists():
             # Set default trace log file if not specified
             if config.trace_log_file is None:

{supernote-0.9.2 → supernote-0.11.0}/supernote/server/constants.py

@@ -17,3 +17,6 @@ CATEGORY_CONTAINERS = {"NOTE", "DOCUMENT"}
 
 # Forced order and specific names for web API root (when flatten=True)
 ORDERED_WEB_ROOT = ["Note", "Document"]
+
+# Blob Storage Buckets
+USER_DATA_BUCKET = "supernote-user-data"

{supernote-0.9.2 → supernote-0.11.0}/supernote/server/db/models/file.py

@@ -39,6 +39,9 @@ class UserFileDO(Base):
     md5: Mapped[Optional[str]] = mapped_column(String, nullable=True)
     """Content hash."""
 
+    storage_key: Mapped[Optional[str]] = mapped_column(String, nullable=True)
+    """Physical storage key (inner_name/UUID)."""
+
     is_active: Mapped[str] = mapped_column(String(1), default="Y", nullable=False)
     """'Y' = Active, 'N' = Deleted."""
 

{supernote-0.9.2 → supernote-0.11.0}/supernote/server/routes/file_device.py

@@ -40,6 +40,7 @@ from supernote.server.exceptions import SupernoteError
 from supernote.server.services.file import (
     FileEntity,
     FileService,
+    generate_inner_name,
 )
 from supernote.server.utils.url_signer import UrlSigner
 
@@ -278,13 +279,21 @@ async def handle_upload_apply(request: web.Request) -> web.Response:
     try:
         url_signer: UrlSigner = request.app["url_signer"]
 
-        encoded_name = urllib.parse.quote(file_name)
+        # Generate a unique inner name for storage
+        inner_name = generate_inner_name(file_name, req_data.equipment_no)
+        encoded_name = urllib.parse.quote(inner_name)
 
-        # Simple Upload URL: /api/oss/upload?object_name={name}
+        # Simple Upload URL: /api/oss/upload?object_name={name}&timestamp={ms}
         simple_path = f"/api/oss/upload?object_name={encoded_name}"
         full_upload_url_path = await url_signer.sign(simple_path, user=request["user"])
         full_upload_url = f"{request.scheme}://{request.host}{full_upload_url_path}"
 
+        # Extract signature and timestamp using UrlSigner helpers
+        signature = UrlSigner.extract_signature(full_upload_url_path)
+        x_amz_date = UrlSigner.extract_timestamp(full_upload_url_path)
+        if not signature or not x_amz_date:
+            raise SupernoteError("Server generated invalid upload URL")
+
         # Part Upload URL: /api/oss/upload/part?object_name={name}
         # Client will append &uploadId=...&partNumber=...
         part_path = f"/api/oss/upload/part?object_name={encoded_name}"
@@ -294,10 +303,10 @@ async def handle_upload_apply(request: web.Request) -> web.Response:
         return web.json_response(
             FileUploadApplyLocalVO(
                 equipment_no=req_data.equipment_no or "",
-                bucket_name="supernote-local",
-                inner_name=file_name,
-                x_amz_date="",
-                authorization="",
+                bucket_name=file_name,  # Reference impl checks this matches filename
+                inner_name=inner_name,
+                x_amz_date=x_amz_date,
+                authorization=signature,
                 full_upload_url=full_upload_url,
                 part_upload_url=part_upload_url,
             ).to_dict()
@@ -318,12 +327,19 @@ async def handle_upload_finish(request: web.Request) -> web.Response:
     user_email = request["user"]
     file_service: FileService = request.app["file_service"]
 
+    if not req_data.inner_name:
+        return web.json_response(
+            create_error_response("Invalid upload missing inner name").to_dict(),
+            status=400,
+        )
+
     try:
         entity = await file_service.finish_upload(
             user_email,
             req_data.file_name,
             req_data.path,
             req_data.content_hash,
+            inner_name=req_data.inner_name,
         )
     except SupernoteError as err:
         return err.to_response()

{supernote-0.9.2 → supernote-0.11.0}/supernote/server/routes/oss.py

@@ -2,12 +2,15 @@
 
 import asyncio
 import logging
+from collections.abc import AsyncGenerator
 
 from aiohttp import BodyPartReader, web
 
 from supernote.models.base import create_error_response
 from supernote.models.system import FileChunkParams, FileChunkVO, UploadFileVO
+from supernote.server.constants import USER_DATA_BUCKET
 from supernote.server.exceptions import SupernoteError
+from supernote.server.services.blob import BlobStorage
 from supernote.server.services.file import FileService
 from supernote.server.utils.url_signer import UrlSigner
 
@@ -17,6 +20,20 @@ logger = logging.getLogger(__name__)
 routes = web.RouteTableDef()
 
 
+def _get_chunk_key(object_name: str, part_number: int) -> str:
+    """Generate a storage key for a file chunk."""
+    return f"{object_name}.part.{part_number}"
+
+
+async def _stream_upload_field(field: BodyPartReader) -> AsyncGenerator[bytes, None]:
+    """Stream chunks from a multipart field."""
+    while True:
+        chunk = await field.read_chunk()
+        if not chunk:
+            break
+        yield chunk
+
+
 @routes.post("/api/oss/upload")
 @public_route
 async def handle_oss_upload(request: web.Request) -> web.Response:
@@ -24,8 +41,8 @@ async def handle_oss_upload(request: web.Request) -> web.Response:
 
     Query: object_name
     """
-    file_service: FileService = request.app["file_service"]
     url_signer: UrlSigner = request.app["url_signer"]
+    blob_storage: BlobStorage = request.app["blob_storage"]
 
     try:
         payload = await url_signer.verify(request.path_qs)
@@ -49,17 +66,17 @@ async def handle_oss_upload(request: web.Request) -> web.Response:
     reader = await request.multipart()
     field = await reader.next()
     if isinstance(field, BodyPartReader) and field.name == "file":
-        total_bytes, md5_hash = await file_service.save_temp_file(
-            user_email, object_name, field.read_chunk
+        metadata = await blob_storage.put(
+            USER_DATA_BUCKET, object_name, _stream_upload_field(field)
         )
         logger.info(
-            f"Received OSS upload for {object_name} (user: {user_email}): {total_bytes} bytes, MD5: {md5_hash}"
+            f"Received OSS upload for {object_name} (user: {user_email}): {metadata.size} bytes, MD5: {metadata.content_md5}"
         )
 
         # Return UploadFileVO with innerName and md5
         response = UploadFileVO(
             inner_name=object_name,
-            md5=md5_hash,
+            md5=metadata.content_md5,
         )
         return web.json_response(response.to_dict())
 
@@ -75,8 +92,8 @@ async def handle_oss_upload_part(request: web.Request) -> web.Response:
     Endpoint: POST /api/oss/upload/part
     Query Params: uploadId, partNumber, object_name, signature, totalChunks (optional/implied for implicit merge)
     """
-    file_service: FileService = request.app["file_service"]
     url_signer: UrlSigner = request.app["url_signer"]
+    blob_storage: BlobStorage = request.app["blob_storage"]
 
     query_dict = dict(request.query)
     try:
@@ -111,13 +128,14 @@ async def handle_oss_upload_part(request: web.Request) -> web.Response:
     reader = await request.multipart()
     field = await reader.next()
     if isinstance(field, BodyPartReader) and field.name == "file":
-        total_bytes, chunk_md5 = await file_service.save_chunk_file(
-            user_email,
-            params.upload_id,
-            params.object_name,
-            params.part_number,
-            field.read_chunk,
+        chunk_key = _get_chunk_key(params.object_name, params.part_number)
+
+        metadata = await blob_storage.put(
+            USER_DATA_BUCKET, chunk_key, _stream_upload_field(field)
         )
+        chunk_md5 = metadata.content_md5
+        total_bytes = metadata.size
+
         logger.info(
             f"Received chunk {params.part_number} for {params.object_name} (uploadId: {params.upload_id}): {total_bytes} bytes, MD5: {chunk_md5}"
         )
@@ -128,17 +146,28 @@ async def handle_oss_upload_part(request: web.Request) -> web.Response:
                 logger.info(
                     f"Implicitly merging {params.total_chunks} chunks for {params.object_name}"
                 )
-                await file_service.merge_chunks(
-                    user_email,
-                    params.upload_id,
-                    params.object_name,
-                    params.total_chunks,
-                )
-                await asyncio.to_thread(
-                    file_service.cleanup_chunks, user_email, params.upload_id
+                source_keys = [
+                    _get_chunk_key(params.object_name, i)
+                    for i in range(1, params.total_chunks + 1)
+                ]
+
+                async def combined_stream() -> AsyncGenerator[bytes, None]:
+                    for source_key in source_keys:
+                        async for chunk in blob_storage.get(
+                            USER_DATA_BUCKET, source_key
+                        ):
+                            yield chunk
+
+                await blob_storage.put(
+                    USER_DATA_BUCKET, params.object_name, combined_stream()
                 )
                 logger.info(f"Successfully merged chunks for {params.object_name}")
 
+                # Cleanup chunks
+                await asyncio.gather(
+                    *[blob_storage.delete(USER_DATA_BUCKET, key) for key in source_keys]
+                )
+
         # Return FileChunkVO with chunk MD5
         resp_vo = FileChunkVO(
             upload_id=params.upload_id,
@@ -194,13 +223,12 @@ async def handle_oss_download(request: web.Request) -> web.StreamResponse:
             create_error_response("Not a file").to_dict(), status=400
         )
 
-    content_hash = info.md5
-    if not content_hash:
+    if not info.storage_key:
         return web.json_response(
             create_error_response("File content not found").to_dict(), status=404
         )
 
-    if not await file_service.blob_storage.exists(content_hash):
+    if not await file_service.blob_storage.exists(USER_DATA_BUCKET, info.storage_key):
         return web.json_response(
             create_error_response("Blob not found").to_dict(), status=404
         )
@@ -252,20 +280,11 @@ async def handle_oss_download(request: web.Request) -> web.StreamResponse:
     await response.prepare(request)
 
     try:
-        async with file_service.blob_storage.open_blob(content_hash) as f:
-            if start > 0:
-                await f.seek(start)
-
-            bytes_to_send = content_length
-            chunk_size = 8192
-
-            while bytes_to_send > 0:
-                read_size = min(chunk_size, bytes_to_send)
-                chunk = await f.read(read_size)
-                if not chunk:
-                    break
-                await response.write(chunk)
-                bytes_to_send -= len(chunk)
+        stream = file_service.blob_storage.get(
+            USER_DATA_BUCKET, info.storage_key, start=start, end=end
+        )
+        async for chunk in stream:
+            await response.write(chunk)
 
     except FileNotFoundError:
         return web.json_response(

supernote-0.11.0/supernote/server/services/blob.py

@@ -0,0 +1,214 @@
+import hashlib
+import secrets
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from pathlib import Path
+from typing import AsyncGenerator
+
+import aiofiles
+import aiofiles.os
+
+
+@dataclass
+class BlobMetadata:
+    """Metadata for a blob."""
+
+    content_type: str | None = None
+    content_md5: str | None = None
+    size: int = 0
+
+
+class BlobStorage(ABC):
+    """Interface for Key-Value Blob Storage."""
+
+    @abstractmethod
+    async def put(
+        self, bucket: str, key: str, stream: AsyncGenerator[bytes, None] | bytes
+    ) -> BlobMetadata:
+        """Write blob to storage."""
+        pass
+
+    @abstractmethod
+    def get(
+        self, bucket: str, key: str, start: int | None = None, end: int | None = None
+    ) -> AsyncGenerator[bytes, None]:
+        """Read blob content.
+
+        Args:
+            bucket: Bucket name.
+            key: Blob key.
+            start: Start byte position (inclusive).
+            end: End byte position (inclusive).
+        """
+        pass
+
+    @abstractmethod
+    async def delete(self, bucket: str, key: str) -> None:
+        """Delete blob."""
+        pass
+
+    @abstractmethod
+    async def exists(self, bucket: str, key: str) -> bool:
+        """Check if blob exists."""
+        pass
+
+    @abstractmethod
+    async def get_metadata(
+        self, bucket: str, key: str, include_md5: bool = False
+    ) -> BlobMetadata:
+        """Get metadata for a blob.
+
+        Args:
+            bucket: Bucket name.
+            key: Blob key.
+            include_md5: If True, compute and return MD5 checksum.
+
+        Returns:
+            BlobMetadata with size and optional content_md5.
+        """
+        pass
+
+    @abstractmethod
+    def get_blob_path(self, bucket: str, key: str) -> Path:
+        """Get physical path to the blob (optional, useful for serving files)."""
+        pass
+
+
+class LocalBlobStorage(BlobStorage):
+    """Local filesystem implementation of Blob Storage.
+
+    Path structure: <root>/<bucket>/<key[0:2]>/<key>
+    """
+
+    def __init__(self, storage_root: Path) -> None:
+        """Create a local blob storage instance."""
+        self.root = storage_root
+        self.root.mkdir(parents=True, exist_ok=True)
+
+    def _get_path(self, bucket: str, key: str) -> Path:
+        """Get physical path to the blob."""
+        # Clean inputs to prevent traversal
+        clean_bucket = Path(bucket).name
+        clean_key = Path(key).name
+        prefix = clean_key[:2] if len(clean_key) >= 2 else "misc"
+        return self.root / clean_bucket / prefix / clean_key
+
+    async def put(
+        self, bucket: str, key: str, stream: AsyncGenerator[bytes, None] | bytes
+    ) -> BlobMetadata:
+        """Write blob to storage."""
+        blob_path = self._get_path(bucket, key)
+        await aiofiles.os.makedirs(blob_path.parent, exist_ok=True)
+
+        # Write to temp file for atomicity
+        temp_dir = self.root / "temp"
+        await aiofiles.os.makedirs(temp_dir, exist_ok=True)
+        temp_path = temp_dir / f"{secrets.token_hex(8)}.tmp"
+
+        total_size = 0
+        md5_hasher = hashlib.md5()
+
+        try:
+            async with aiofiles.open(temp_path, "wb") as f:
+                if isinstance(stream, bytes):
+                    total_size = len(stream)
+                    md5_hasher.update(stream)
+                    await f.write(stream)
+                else:
+                    async for chunk in stream:
+                        total_size += len(chunk)
+                        md5_hasher.update(chunk)
+                        await f.write(chunk)
+
+            # Move to final location
+            await aiofiles.os.rename(temp_path, blob_path)
+
+            return BlobMetadata(
+                content_md5=md5_hasher.hexdigest(),
+                size=total_size,
+            )
+
+        except Exception:
+            if await aiofiles.os.path.exists(temp_path):
+                await aiofiles.os.remove(temp_path)
+            raise
+
+    async def get(
+        self, bucket: str, key: str, start: int | None = None, end: int | None = None
+    ) -> AsyncGenerator[bytes, None]:
+        """Read blob content."""
+        path = self._get_path(bucket, key)
+        if not await aiofiles.os.path.exists(path):
+            raise FileNotFoundError(f"Blob {bucket}/{key} not found")
+
+        bytes_remaining = None
+        if end is not None:
+            if start is None:
+                start = 0
+            bytes_remaining = end - start + 1
+
+        async with aiofiles.open(path, "rb") as f:
+            if start is not None and start > 0:
+                await f.seek(start)
+
+            while True:
+                chunk_size = 8192
+                if bytes_remaining is not None:
+                    if bytes_remaining <= 0:
+                        break
+                    chunk_size = min(chunk_size, bytes_remaining)
+
+                chunk = await f.read(chunk_size)
+                if not chunk:
+                    break
+
+                if bytes_remaining is not None:
+                    bytes_remaining -= len(chunk)
+
+                yield chunk
+
+    async def delete(self, bucket: str, key: str) -> None:
+        """Delete blob."""
+        path = self._get_path(bucket, key)
+        if await aiofiles.os.path.exists(path):
+            await aiofiles.os.remove(path)
+
+    async def exists(self, bucket: str, key: str) -> bool:
+        """Check if blob exists."""
+        return bool(await aiofiles.os.path.exists(self._get_path(bucket, key)))
+
+    async def get_metadata(
+        self, bucket: str, key: str, include_md5: bool = False
+    ) -> BlobMetadata:
+        """Get metadata for a blob."""
+        path = self._get_path(bucket, key)
+        if not await aiofiles.os.path.exists(path):
+            raise FileNotFoundError(f"Blob {bucket}/{key} not found")
+
+        stat = await aiofiles.os.stat(path)
+
+        if not include_md5:
+            return BlobMetadata(size=stat.st_size)
+
+        # Compute MD5 and size
+        md5_hasher = hashlib.md5()
+        read_size = 0
+        async with aiofiles.open(path, "rb") as f:
+            while True:
+                chunk = await f.read(8192)
+                if not chunk:
+                    break
+                md5_hasher.update(chunk)
+                read_size += len(chunk)
+
+        if read_size != stat.st_size:
+            # This could happen if file was modified during read
+            raise ValueError(
+                f"File size changed during read: metadata={stat.st_size}, read={read_size}"
+            )
+
+        return BlobMetadata(size=stat.st_size, content_md5=md5_hasher.hexdigest())
+
+    def get_blob_path(self, bucket: str, key: str) -> Path:
+        """Get physical path to the blob."""
+        return self._get_path(bucket, key)