amsdal 0.5.7__cp312-cp312-macosx_10_13_universal2.whl → 0.5.8__cp312-cp312-macosx_10_13_universal2.whl

amsdal/storages/__init__.py

@@ -0,0 +1,20 @@
+from amsdal_models.storage.base import Storage
+
+from amsdal.configs.main import settings
+
+_DEFAULT_STORAGE = None
+
+
+def set_default_storage(storage: Storage) -> None:
+    global _DEFAULT_STORAGE  # noqa: PLW0603
+    _DEFAULT_STORAGE = storage
+
+
+def default_storage() -> Storage:
+    global _DEFAULT_STORAGE  # noqa: PLW0603
+
+    if _DEFAULT_STORAGE is None:
+        # Determine backend from settings
+        class_path = settings.AMSDAL_DEFAULT_FILE_STORAGE
+        _DEFAULT_STORAGE = Storage.from_storage_spec({'storage_class': class_path})
+    return _DEFAULT_STORAGE

amsdal/storages/__init__.pyi

@@ -0,0 +1,8 @@
+from _typeshed import Incomplete
+from amsdal.configs.main import settings as settings
+from amsdal_models.storage.base import Storage
+
+_DEFAULT_STORAGE: Incomplete
+
+def set_default_storage(storage: Storage) -> None: ...
+def default_storage() -> Storage: ...

amsdal/storages/file_system.py

@@ -0,0 +1,214 @@
+import asyncio
+import os
+from contextlib import suppress
+from pathlib import Path
+from typing import IO
+from typing import Any
+from typing import BinaryIO
+
+from amsdal_models.storage.base import Storage
+from amsdal_models.storage.errors import StorageError
+from amsdal_models.storage.helpers import build_storage_address
+from amsdal_models.storage.types import FileProtocol
+from amsdal_utils.config.manager import AmsdalConfigManager
+
+CHUNK_SIZE = 8 * 1024
+
+
+class FileSystemStorage(Storage):
+    """
+    Simple filesystem-based storage backend.
+
+    - base_dir: root directory for stored files
+    - base_url: URL prefix for building public URLs (optional)
+    """
+
+    keeps_local_copy = False
+
+    def __init__(
+        self,
+        base_dir: str | os.PathLike[Any] | None = None,
+        base_url: str | None = None,
+        *,
+        serialize_base_dir: bool = True,
+        serialize_base_url: bool = True,
+    ) -> None:
+        from amsdal.configs.main import settings
+
+        # If AMSDAL is configured to run in async mode, ensure aiofiles is installed.
+        if AmsdalConfigManager().get_config().async_mode:
+            try:
+                import aiofiles  # noqa: F401
+            except ImportError as e:
+                msg = (
+                    "AMSDAL is configured to run in async mode, but the 'aiofiles' package is not installed.\n"
+                    "Please install it to enable async file operations.\n\n"
+                    "Example:\n"
+                    "    pip install aiofiles\n\n"
+                    f"Original error: {e}"
+                )
+                raise ImportError(msg) from e
+
+        _base_dir = base_dir or settings.AMSDAL_MEDIA_ROOT
+        _base_url = base_url or settings.AMSDAL_MEDIA_URL
+
+        self.base_dir = Path(_base_dir).resolve()
+        self.base_url = _base_url
+        self.base_dir.mkdir(parents=True, exist_ok=True)
+        self.serialize_base_dir = serialize_base_dir
+        self.serialize_base_url = serialize_base_url
+
+    def save(self, file: FileProtocol, content: BinaryIO) -> str:
+        suggested = file.filename
+        final_name = self._get_available_name(suggested)
+        full_path = self._full_path(final_name)
+        full_path.parent.mkdir(parents=True, exist_ok=True)
+
+        with full_path.open('wb') as f:
+            # Ensure reading from start
+            with suppress(Exception):
+                if hasattr(content, 'seek'):
+                    content.seek(0)
+
+            chunk = content.read(CHUNK_SIZE)
+
+            while chunk:
+                f.write(chunk)
+                chunk = content.read(CHUNK_SIZE)
+
+        file.storage_address = build_storage_address(self, final_name)
+        return final_name
+
+    def open(self, file: FileProtocol, mode: str = 'rb') -> IO[Any]:
+        name = self._name_for(file)
+        full_path = self._full_path(name)
+
+        if not full_path.exists():
+            msg = f'File not found: {name}'
+            raise StorageError(msg)
+        return full_path.open(mode)
+
+    def delete(self, file: FileProtocol) -> None:
+        name = self._name_for(file)
+        try:
+            self._full_path(name).unlink(missing_ok=True)
+        except Exception as e:
+            msg = f"Failed to delete '{name}': {e}"
+            raise StorageError(msg) from e
+
+    def exists(self, file: FileProtocol) -> bool:
+        name = self._name_for(file)
+        return self._full_path(name).exists()
+
+    def url(self, file: FileProtocol) -> str:
+        name = self._name_for(file)
+        if not self.base_url:
+            # Return file path as fallback
+            return self._full_path(name).as_uri()
+
+        prefix = self.base_url.rstrip('/')
+        name_posix = Path(name).as_posix().lstrip('/')
+
+        return f'{prefix}/{name_posix}'
+
+    # ---- async counterparts ----
+    async def asave(self, file: FileProtocol, content: BinaryIO) -> str:
+        import aiofiles.os
+
+        suggested = file.filename
+        final_name = self._get_available_name(suggested)
+        full_path = self._full_path(final_name)
+        # Ensure directory exists
+        await aiofiles.os.makedirs(str(full_path.parent), exist_ok=True)
+
+        # Ensure reading from start
+        with suppress(Exception):
+            if hasattr(content, 'seek'):
+                with suppress(Exception):
+                    await asyncio.to_thread(content.seek, 0)
+
+        async with aiofiles.open(str(full_path), 'wb') as f:  # type: ignore[call-arg]
+            while True:
+                chunk = await asyncio.to_thread(content.read, CHUNK_SIZE)
+                if not chunk:
+                    break
+                await f.write(chunk)
+
+        file.storage_address = build_storage_address(self, final_name)
+
+        return final_name
+
+    async def aopen(self, file: FileProtocol, mode: str = 'rb') -> Any:
+        import aiofiles.ospath
+
+        name = self._name_for(file)
+        full_path = self._full_path(name)
+
+        if not await aiofiles.ospath.exists(str(full_path)):
+            msg = f'File not found: {name}'
+            raise StorageError(msg)
+
+        return aiofiles.open(str(full_path), mode)  # type: ignore[call-overload]
+
+    async def adelete(self, file: FileProtocol) -> None:
+        import aiofiles.os
+        import aiofiles.ospath
+
+        name = self._name_for(file)
+        full_path = self._full_path(name)
+
+        if await aiofiles.ospath.exists(str(full_path)):
+            try:
+                await aiofiles.os.remove(str(full_path))
+            except Exception as e:  # pragma: no cover - error path
+                msg = f"Failed to delete '{name}': {e}"
+                raise StorageError(msg) from e
+
+    async def aexists(self, file: FileProtocol) -> bool:
+        import aiofiles.ospath
+
+        name = self._name_for(file)
+        return await aiofiles.ospath.exists(str(self._full_path(name)))
+
+    async def aurl(self, file: FileProtocol) -> str:
+        # Pure computation; no disk I/O.
+        return self.url(file)
+
+    def _full_path(self, name: str) -> Path:
+        # Sanitize name to avoid path traversal
+        norm = Path(name).as_posix().lstrip('/')
+
+        return (self.base_dir / norm).resolve()
+
+    def _get_available_name(self, name: str) -> str:
+        # If file exists, add suffixes to avoid collision
+        candidate = Path(name)
+        base = candidate.stem
+        suffix = candidate.suffix
+        parent = candidate.parent.as_posix()
+        i = 0
+        final = candidate.as_posix()
+
+        # Check filesystem directly to avoid relying on public exists signature
+        while self._full_path(final).exists():
+            i += 1
+            new_name = f'{base}_{i}{suffix}'
+            final = str(Path(parent) / new_name) if parent and parent != '.' else new_name
+        return final
+
+    def _name_for(self, file: FileProtocol) -> str:
+        if getattr(file, 'storage_address', None) and getattr(file.storage_address, 'ref', None) is not None:
+            if getattr(file.storage_address.ref, 'object_id', None) is not None:  # type: ignore[union-attr]
+                return str(file.storage_address.ref.object_id)  # type: ignore[union-attr]
+        return file.filename
+
+    def _export_kwargs(self) -> dict[str, Any]:
+        kwargs = {}
+
+        if self.serialize_base_dir:
+            kwargs['base_dir'] = str(self.base_dir)
+
+        if self.serialize_base_url:
+            kwargs['base_url'] = self.base_url
+
+        return kwargs

amsdal/storages/file_system.pyi

@@ -0,0 +1,36 @@
+import os
+from _typeshed import Incomplete
+from amsdal_models.storage.base import Storage
+from amsdal_models.storage.types import FileProtocol
+from pathlib import Path
+from typing import Any, BinaryIO, IO
+
+CHUNK_SIZE: Incomplete
+
+class FileSystemStorage(Storage):
+    """
+    Simple filesystem-based storage backend.
+
+    - base_dir: root directory for stored files
+    - base_url: URL prefix for building public URLs (optional)
+    """
+    keeps_local_copy: bool
+    base_dir: Incomplete
+    base_url: Incomplete
+    serialize_base_dir: Incomplete
+    serialize_base_url: Incomplete
+    def __init__(self, base_dir: str | os.PathLike[Any] | None = None, base_url: str | None = None, *, serialize_base_dir: bool = True, serialize_base_url: bool = True) -> None: ...
+    def save(self, file: FileProtocol, content: BinaryIO) -> str: ...
+    def open(self, file: FileProtocol, mode: str = 'rb') -> IO[Any]: ...
+    def delete(self, file: FileProtocol) -> None: ...
+    def exists(self, file: FileProtocol) -> bool: ...
+    def url(self, file: FileProtocol) -> str: ...
+    async def asave(self, file: FileProtocol, content: BinaryIO) -> str: ...
+    async def aopen(self, file: FileProtocol, mode: str = 'rb') -> Any: ...
+    async def adelete(self, file: FileProtocol) -> None: ...
+    async def aexists(self, file: FileProtocol) -> bool: ...
+    async def aurl(self, file: FileProtocol) -> str: ...
+    def _full_path(self, name: str) -> Path: ...
+    def _get_available_name(self, name: str) -> str: ...
+    def _name_for(self, file: FileProtocol) -> str: ...
+    def _export_kwargs(self) -> dict[str, Any]: ...

amsdal/utils/tests/migrations.py

@@ -1,17 +1,19 @@
 import contextlib
+from asyncio import iscoroutine
 
 from amsdal_data.connections.historical.schema_version_manager import AsyncHistoricalSchemaVersionManager
 from amsdal_data.connections.historical.schema_version_manager import HistoricalSchemaVersionManager
 from amsdal_models.migration import migrations
 from amsdal_models.migration.executors.default_executor import DefaultAsyncMigrationExecutor
 from amsdal_models.migration.executors.default_executor import DefaultMigrationExecutor
-from amsdal_models.migration.file_migration_executor import SimpleFileMigrationExecutorManager
+from amsdal_models.migration.file_migration_executor import AsyncFileMigrationExecutorManager
+from amsdal_models.migration.file_migration_executor import FileMigrationExecutorManager
 from amsdal_models.migration.file_migration_generator import SimpleFileMigrationGenerator
+from amsdal_models.migration.file_migration_store import AsyncFileMigrationStore
+from amsdal_models.migration.file_migration_store import FileMigrationStore
 from amsdal_models.migration.file_migration_writer import FileMigrationWriter
-from amsdal_models.migration.migrations import MigrateData
 from amsdal_models.migration.migrations import MigrationSchemas
 from amsdal_models.migration.migrations_loader import MigrationsLoader
-from amsdal_models.migration.utils import contrib_to_module_root_path
 from amsdal_models.schemas.class_schema_loader import ClassSchemaLoader
 from amsdal_utils.models.enums import ModuleType
 
@@ -22,29 +24,28 @@ from amsdal.configs.main import settings
 def migrate() -> None:
     schemas = MigrationSchemas()
     executor = DefaultMigrationExecutor(schemas, use_foreign_keys=True)
+    store = FileMigrationStore(settings.migrations_root_path)
+    store.init_migration_table()
+
+    class UserMigrationsLoader(MigrationsLoader):
+        def __init__(self):
+            super().__init__(settings.migrations_root_path, ModuleType.USER)
+            self._migrations_files = []
 
     with contextlib.suppress(Exception):
         HistoricalSchemaVersionManager().object_classes  # noqa: B018
 
-    _migrate_per_loader(
-        executor,
-        MigrationsLoader(
-            migrations_dir=CORE_MIGRATIONS_PATH,
-            module_type=ModuleType.CORE,
-        ),
+    # migrate core and contrib due to applied migrations
+    executor_manager = FileMigrationExecutorManager(
+        core_migrations_path=CORE_MIGRATIONS_PATH,
+        app_migrations_loader=UserMigrationsLoader(),
+        executor=executor,
+        store=store,
+        contrib=settings.CONTRIBS,
     )
+    executor_manager.execute()
 
-    for contrib in settings.CONTRIBS:
-        contrib_root_path = contrib_to_module_root_path(contrib)
-        _migrate_per_loader(
-            executor,
-            MigrationsLoader(
-                migrations_dir=contrib_root_path / settings.MIGRATIONS_DIRECTORY_NAME,
-                module_type=ModuleType.CONTRIB,
-                module_name=contrib,
-            ),
-        )
-
+    # always apply migrations for user models
     user_schema_loader = ClassSchemaLoader(
         settings.USER_MODELS_MODULE,
         class_filter=lambda cls: cls.__module_type__ == ModuleType.USER,
@@ -85,46 +86,31 @@ def migrate() -> None:
     executor.flush_buffer()
 
 
-def _migrate_per_loader(executor: DefaultMigrationExecutor, loader: MigrationsLoader) -> None:
-    for _migration in loader:
-        migration_class = SimpleFileMigrationExecutorManager.get_migration_class(_migration)
-        migration_class_instance = migration_class()
-
-        for _operation in migration_class_instance.operations:
-            if isinstance(_operation, MigrateData):
-                executor.flush_buffer()
-
-            _operation.forward(executor)
-
-        executor.flush_buffer()
-
-
 async def async_migrate() -> None:
     schemas = MigrationSchemas()
     executor = DefaultAsyncMigrationExecutor(schemas)
+    store = AsyncFileMigrationStore(settings.migrations_root_path)
+    await store.init_migration_table()
+
+    class UserMigrationsLoader(MigrationsLoader):
+        def __init__(self):
+            super().__init__(settings.migrations_root_path, ModuleType.USER)
+            self._migrations_files = []
 
     with contextlib.suppress(Exception):
         await AsyncHistoricalSchemaVersionManager().object_classes
 
-    await _async_migrate_per_loader(
-        executor,
-        MigrationsLoader(
-            migrations_dir=CORE_MIGRATIONS_PATH,
-            module_type=ModuleType.CORE,
-        ),
+    # migrate core and contrib due to applied migrations
+    executor_manager = AsyncFileMigrationExecutorManager(
+        core_migrations_path=CORE_MIGRATIONS_PATH,
+        app_migrations_loader=UserMigrationsLoader(),
+        executor=executor,
+        store=store,
+        contrib=settings.CONTRIBS,
     )
+    await executor_manager.execute()
 
-    for contrib in settings.CONTRIBS:
-        contrib_root_path = contrib_to_module_root_path(contrib)
-        await _async_migrate_per_loader(
-            executor,
-            MigrationsLoader(
-                migrations_dir=contrib_root_path / settings.MIGRATIONS_DIRECTORY_NAME,
-                module_type=ModuleType.CONTRIB,
-                module_name=contrib,
-            ),
-        )
-
+    # always apply migrations for user models
     user_schema_loader = ClassSchemaLoader(
         settings.USER_MODELS_MODULE,
         class_filter=lambda cls: cls.__module_type__ == ModuleType.USER,
@@ -146,6 +132,10 @@ async def async_migrate() -> None:
             )
 
             _operation.forward(executor)
+            forward_result = _operation.forward(executor)
+
+            if iscoroutine(forward_result):
+                await forward_result
 
     for object_schema in _cycle_schemas:
         for _operation_data in SimpleFileMigrationGenerator.build_operations(
@@ -160,20 +150,9 @@ async def async_migrate() -> None:
                 new_schema=_operation_data.new_schema.model_dump(),
             )
 
-            _operation.forward(executor)
-
-    await executor.flush_buffer()
-
+            forward_result = _operation.forward(executor)
 
-async def _async_migrate_per_loader(executor: DefaultAsyncMigrationExecutor, loader: MigrationsLoader) -> None:
-    for _migration in loader:
-        migration_class = SimpleFileMigrationExecutorManager.get_migration_class(_migration)
-        migration_class_instance = migration_class()
+            if iscoroutine(forward_result):
+                await forward_result
 
-        for _operation in migration_class_instance.operations:
-            if isinstance(_operation, MigrateData):
-                await executor.flush_buffer()
-
-            _operation.forward(executor)
-
-        await executor.flush_buffer()
+    await executor.flush_buffer()

{amsdal-0.5.7.dist-info → amsdal-0.5.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: amsdal
-Version: 0.5.7
+Version: 0.5.8
 Summary: AMSDAL
 License: AMSDAL End User License Agreement