amsdal 0.3.3__cp311-cp311-macosx_10_9_universal2.whl → 0.5.29__cp311-cp311-macosx_10_9_universal2.whl

amsdal/storages/file_system.py

@@ -0,0 +1,214 @@
+import asyncio
+import os
+from contextlib import suppress
+from pathlib import Path
+from typing import IO
+from typing import Any
+from typing import BinaryIO
+
+from amsdal_models.storage.base import Storage
+from amsdal_models.storage.errors import StorageError
+from amsdal_models.storage.helpers import build_storage_address
+from amsdal_models.storage.types import FileProtocol
+from amsdal_utils.config.manager import AmsdalConfigManager
+
+CHUNK_SIZE = 8 * 1024
+
+
+class FileSystemStorage(Storage):
+    """
+    Simple filesystem-based storage backend.
+
+    - base_dir: root directory for stored files
+    - base_url: URL prefix for building public URLs (optional)
+    """
+
+    keeps_local_copy = False
+
+    def __init__(
+        self,
+        base_dir: str | os.PathLike[Any] | None = None,
+        base_url: str | None = None,
+        *,
+        serialize_base_dir: bool = True,
+        serialize_base_url: bool = True,
+    ) -> None:
+        from amsdal.configs.main import settings
+
+        # If AMSDAL is configured to run in async mode, ensure aiofiles is installed.
+        if AmsdalConfigManager().get_config().async_mode:
+            try:
+                import aiofiles  # noqa: F401
+            except ImportError as e:
+                msg = (
+                    "AMSDAL is configured to run in async mode, but the 'aiofiles' package is not installed.\n"
+                    'Please install it to enable async file operations.\n\n'
+                    'Example:\n'
+                    '    pip install aiofiles\n\n'
+                    f'Original error: {e}'
+                )
+                raise ImportError(msg) from e
+
+        _base_dir = base_dir or settings.MEDIA_ROOT
+        _base_url = base_url or settings.MEDIA_URL
+
+        self.base_dir = Path(_base_dir).resolve()
+        self.base_url = _base_url
+        self.base_dir.mkdir(parents=True, exist_ok=True)
+        self.serialize_base_dir = serialize_base_dir
+        self.serialize_base_url = serialize_base_url
+
+    def save(self, file: FileProtocol, content: BinaryIO) -> str:
+        suggested = file.filename
+        final_name = self._get_available_name(suggested)
+        full_path = self._full_path(final_name)
+        full_path.parent.mkdir(parents=True, exist_ok=True)
+
+        with full_path.open('wb') as f:
+            # Ensure reading from start
+            with suppress(Exception):
+                if hasattr(content, 'seek'):
+                    content.seek(0)
+
+            chunk = content.read(CHUNK_SIZE)
+
+            while chunk:
+                f.write(chunk)
+                chunk = content.read(CHUNK_SIZE)
+
+        file.storage_address = build_storage_address(self, final_name)
+        return final_name
+
+    def open(self, file: FileProtocol, mode: str = 'rb') -> IO[Any]:
+        name = self._name_for(file)
+        full_path = self._full_path(name)
+
+        if not full_path.exists():
+            msg = f'File not found: {name}'
+            raise StorageError(msg)
+        return full_path.open(mode)
+
+    def delete(self, file: FileProtocol) -> None:
+        name = self._name_for(file)
+        try:
+            self._full_path(name).unlink(missing_ok=True)
+        except Exception as e:
+            msg = f"Failed to delete '{name}': {e}"
+            raise StorageError(msg) from e
+
+    def exists(self, file: FileProtocol) -> bool:
+        name = self._name_for(file)
+        return self._full_path(name).exists()
+
+    def url(self, file: FileProtocol) -> str:
+        name = self._name_for(file)
+        if not self.base_url:
+            # Return file path as fallback
+            return self._full_path(name).as_uri()
+
+        prefix = self.base_url.rstrip('/')
+        name_posix = Path(name).as_posix().lstrip('/')
+
+        return f'{prefix}/{name_posix}'
+
+    # ---- async counterparts ----
+    async def asave(self, file: FileProtocol, content: BinaryIO) -> str:
+        import aiofiles.os
+
+        suggested = file.filename
+        final_name = self._get_available_name(suggested)
+        full_path = self._full_path(final_name)
+        # Ensure directory exists
+        await aiofiles.os.makedirs(str(full_path.parent), exist_ok=True)
+
+        # Ensure reading from start
+        with suppress(Exception):
+            if hasattr(content, 'seek'):
+                with suppress(Exception):
+                    await asyncio.to_thread(content.seek, 0)
+
+        async with aiofiles.open(str(full_path), 'wb') as f:  # type: ignore[call-arg]
+            while True:
+                chunk = await asyncio.to_thread(content.read, CHUNK_SIZE)
+                if not chunk:
+                    break
+                await f.write(chunk)
+
+        file.storage_address = build_storage_address(self, final_name)
+
+        return final_name
+
+    async def aopen(self, file: FileProtocol, mode: str = 'rb') -> Any:
+        import aiofiles.ospath
+
+        name = self._name_for(file)
+        full_path = self._full_path(name)
+
+        if not await aiofiles.ospath.exists(str(full_path)):
+            msg = f'File not found: {name}'
+            raise StorageError(msg)
+
+        return aiofiles.open(str(full_path), mode)  # type: ignore[call-overload]
+
+    async def adelete(self, file: FileProtocol) -> None:
+        import aiofiles.os
+        import aiofiles.ospath
+
+        name = self._name_for(file)
+        full_path = self._full_path(name)
+
+        if await aiofiles.ospath.exists(str(full_path)):
+            try:
+                await aiofiles.os.remove(str(full_path))
+            except Exception as e:  # pragma: no cover - error path
+                msg = f"Failed to delete '{name}': {e}"
+                raise StorageError(msg) from e
+
+    async def aexists(self, file: FileProtocol) -> bool:
+        import aiofiles.ospath
+
+        name = self._name_for(file)
+        return await aiofiles.ospath.exists(str(self._full_path(name)))
+
+    async def aurl(self, file: FileProtocol) -> str:
+        # Pure computation; no disk I/O.
+        return self.url(file)
+
+    def _full_path(self, name: str) -> Path:
+        # Sanitize name to avoid path traversal
+        norm = Path(name).as_posix().lstrip('/')
+
+        return (self.base_dir / norm).resolve()
+
+    def _get_available_name(self, name: str) -> str:
+        # If file exists, add suffixes to avoid collision
+        candidate = Path(name)
+        base = candidate.stem
+        suffix = candidate.suffix
+        parent = candidate.parent.as_posix()
+        i = 0
+        final = candidate.as_posix()
+
+        # Check filesystem directly to avoid relying on public exists signature
+        while self._full_path(final).exists():
+            i += 1
+            new_name = f'{base}_{i}{suffix}'
+            final = str(Path(parent) / new_name) if parent and parent != '.' else new_name
+        return final
+
+    def _name_for(self, file: FileProtocol) -> str:
+        if getattr(file, 'storage_address', None) and getattr(file.storage_address, 'ref', None) is not None:
+            if getattr(file.storage_address.ref, 'object_id', None) is not None:  # type: ignore[union-attr]
+                return str(file.storage_address.ref.object_id)  # type: ignore[union-attr]
+        return file.filename
+
+    def _export_kwargs(self) -> dict[str, Any]:
+        kwargs = {}
+
+        if self.serialize_base_dir:
+            kwargs['base_dir'] = str(self.base_dir)
+
+        if self.serialize_base_url:
+            kwargs['base_url'] = self.base_url
+
+        return kwargs

amsdal/storages/file_system.pyi

@@ -0,0 +1,36 @@
+import os
+from _typeshed import Incomplete
+from amsdal_models.storage.base import Storage
+from amsdal_models.storage.types import FileProtocol
+from pathlib import Path
+from typing import Any, BinaryIO, IO
+
+CHUNK_SIZE: Incomplete
+
+class FileSystemStorage(Storage):
+    """
+    Simple filesystem-based storage backend.
+
+    - base_dir: root directory for stored files
+    - base_url: URL prefix for building public URLs (optional)
+    """
+    keeps_local_copy: bool
+    base_dir: Incomplete
+    base_url: Incomplete
+    serialize_base_dir: Incomplete
+    serialize_base_url: Incomplete
+    def __init__(self, base_dir: str | os.PathLike[Any] | None = None, base_url: str | None = None, *, serialize_base_dir: bool = True, serialize_base_url: bool = True) -> None: ...
+    def save(self, file: FileProtocol, content: BinaryIO) -> str: ...
+    def open(self, file: FileProtocol, mode: str = 'rb') -> IO[Any]: ...
+    def delete(self, file: FileProtocol) -> None: ...
+    def exists(self, file: FileProtocol) -> bool: ...
+    def url(self, file: FileProtocol) -> str: ...
+    async def asave(self, file: FileProtocol, content: BinaryIO) -> str: ...
+    async def aopen(self, file: FileProtocol, mode: str = 'rb') -> Any: ...
+    async def adelete(self, file: FileProtocol) -> None: ...
+    async def aexists(self, file: FileProtocol) -> bool: ...
+    async def aurl(self, file: FileProtocol) -> str: ...
+    def _full_path(self, name: str) -> Path: ...
+    def _get_available_name(self, name: str) -> str: ...
+    def _name_for(self, file: FileProtocol) -> str: ...
+    def _export_kwargs(self) -> dict[str, Any]: ...

amsdal/transactions/__init__.py

@@ -0,0 +1,13 @@
+from amsdal_data.transactions import async_transaction
+from amsdal_data.transactions import transaction
+from amsdal_data.transactions.background.schedule import SCHEDULE_TYPE
+from amsdal_data.transactions.background.schedule import Crontab
+from amsdal_data.transactions.background.schedule import ScheduleConfig
+
+__all__ = [
+    'SCHEDULE_TYPE',
+    'Crontab',
+    'ScheduleConfig',
+    'async_transaction',
+    'transaction',
+]

amsdal/transactions/__init__.pyi

@@ -0,0 +1,4 @@
+from amsdal_data.transactions import async_transaction as async_transaction, transaction as transaction
+from amsdal_data.transactions.background.schedule import Crontab as Crontab, SCHEDULE_TYPE as SCHEDULE_TYPE, ScheduleConfig as ScheduleConfig
+
+__all__ = ['SCHEDULE_TYPE', 'Crontab', 'ScheduleConfig', 'async_transaction', 'transaction']

amsdal/utils/rollback/__init__.py

@@ -4,7 +4,7 @@ from amsdal_data.application import DataApplication
 from amsdal_data.transactions.decorators import async_transaction
 from amsdal_data.transactions.decorators import transaction
 from amsdal_data.transactions.errors import AmsdalTransactionError
-from amsdal_models.classes.manager import ClassManager
+from amsdal_models.classes.class_manager import ClassManager
 from amsdal_models.querysets.executor import LAKEHOUSE_DB_ALIAS
 
 
@@ -28,14 +28,21 @@ def rollback_to_timestamp(timestamp: float) -> None:
             table=glue.SchemaReference(name='Metadata', version=glue.Version.LATEST),
             where=glue.Conditions(
                 glue.Condition(
-                    field=glue.FieldReference(field=glue.Field(name='updated_at'), table_name='Metadata'),
+                    left=glue.FieldReferenceExpression(
+                        field_reference=glue.FieldReference(field=glue.Field(name='updated_at'), table_name='Metadata'),
+                    ),
                     lookup=glue.FieldLookup.GT,
-                    value=glue.Value(timestamp),
+                    right=glue.Value(timestamp),
                 ),
                 glue.Condition(
-                    field=glue.FieldReference(field=glue.Field(name='prior_version'), table_name='Metadata'),
+                    left=glue.FieldReferenceExpression(
+                        field_reference=glue.FieldReference(
+                            field=glue.Field(name='prior_version'),
+                            table_name='Metadata',
+                        ),
+                    ),
                     lookup=glue.FieldLookup.ISNULL,
-                    value=glue.Value(True),
+                    right=glue.Value(True),
                 ),
             ),
         )
@@ -48,14 +55,21 @@ def rollback_to_timestamp(timestamp: float) -> None:
             table=glue.SchemaReference(name='Metadata', version=glue.Version.LATEST),
             where=glue.Conditions(
                 glue.Condition(
-                    field=glue.FieldReference(field=glue.Field(name='updated_at'), table_name='Metadata'),
+                    left=glue.FieldReferenceExpression(
+                        field_reference=glue.FieldReference(field=glue.Field(name='updated_at'), table_name='Metadata'),
+                    ),
                     lookup=glue.FieldLookup.GT,
-                    value=glue.Value(timestamp),
+                    right=glue.Value(timestamp),
                 ),
                 glue.Condition(
-                    field=glue.FieldReference(field=glue.Field(name='prior_version'), table_name='Metadata'),
+                    left=glue.FieldReferenceExpression(
+                        field_reference=glue.FieldReference(
+                            field=glue.Field(name='prior_version'),
+                            table_name='Metadata',
+                        ),
+                    ),
                     lookup=glue.FieldLookup.ISNULL,
-                    value=glue.Value(False),
+                    right=glue.Value(False),
                 ),
             ),
         )
@@ -83,9 +97,12 @@ def rollback_to_timestamp(timestamp: float) -> None:
             _parent_field.child.child.parent = _parent_field.child  # type: ignore[union-attr]
             _conditions.append(
                 glue.Condition(
-                    field=glue.FieldReference(field=_parent_field, table_name='Metadata'),
+                    left=glue.FieldReferenceExpression(
+                        field_reference=glue.FieldReference(field=_parent_field, table_name='Metadata'),
+                        output_type=str,
+                    ),
                     lookup=glue.FieldLookup.EQ,
-                    value=glue.Value(transaction_id),
+                    right=glue.Value(transaction_id, output_type=str),
                 )
             )
 
@@ -94,9 +111,14 @@ def rollback_to_timestamp(timestamp: float) -> None:
                 table=glue.SchemaReference(name='Metadata', version=glue.Version.LATEST),
                 where=glue.Conditions(
                     glue.Condition(
-                        field=glue.FieldReference(field=glue.Field(name='updated_at'), table_name='Metadata'),
+                        left=glue.FieldReferenceExpression(
+                            field_reference=glue.FieldReference(
+                                field=glue.Field(name='updated_at'),
+                                table_name='Metadata',
+                            ),
+                        ),
                         lookup=glue.FieldLookup.LTE,
-                        value=glue.Value(timestamp),
+                        right=glue.Value(timestamp),
                     ),
                     glue.Conditions(*_conditions, connector=glue.FilterConnector.OR),
                 ),
@@ -108,9 +130,7 @@ def rollback_to_timestamp(timestamp: float) -> None:
 
     for m in metadatas_to_delete:
         class_name = m.data['class_schema_reference']['ref']['object_id']
-        schema_type = class_manager.resolve_schema_type(class_name)
-
-        model_class = class_manager.import_model_class(class_name, schema_type)
+        model_class = class_manager.import_class(class_name)
         obj = (
             model_class.objects.filter(_address__object_id=m.data['object_id'])
             .using(LAKEHOUSE_DB_ALIAS)
@@ -123,8 +143,7 @@ def rollback_to_timestamp(timestamp: float) -> None:
             obj.delete()
 
     for object_id, class_name in ids_to_revert:
-        schema_type = class_manager.resolve_schema_type(class_name)
-        model_class = class_manager.import_model_class(class_name, schema_type)
+        model_class = class_manager.import_class(class_name)
 
         obj = (
             model_class.objects.filter(_address__object_id=object_id)
@@ -133,13 +152,14 @@ def rollback_to_timestamp(timestamp: float) -> None:
             .first()
             .execute()
         )
-        old_obj = (
-            model_class.objects.filter(_address__object_id=object_id, _metadata__updated_at__lte=timestamp)
-            .using(LAKEHOUSE_DB_ALIAS)
-            .order_by('-_metadata__updated_at')
-            .first()
-            .execute()
-        )
+        old_obj = obj.previous_version()  # type: ignore[union-attr]
+        # old_obj = (
+        #     model_class.objects.filter(_address__object_id=object_id, _metadata__updated_at__lte=timestamp)
+        #     .using(LAKEHOUSE_DB_ALIAS)
+        #     .order_by('-_metadata__updated_at')
+        #     .first()
+        #     .execute()
+        # )
 
         if obj and old_obj:
             for field, value in old_obj.model_dump().items():
@@ -180,9 +200,12 @@ def rollback_transaction(transaction_id: str) -> None:
             table=glue.SchemaReference(name='Metadata', version=glue.Version.LATEST),
             where=glue.Conditions(
                 glue.Condition(
-                    field=glue.FieldReference(field=_parent_field, table_name='Metadata'),
+                    left=glue.FieldReferenceExpression(
+                        field_reference=glue.FieldReference(field=_parent_field, table_name='Metadata'),
+                        output_type=str,
+                    ),
                     lookup=glue.FieldLookup.EQ,
-                    value=glue.Value(transaction_id),
+                    right=glue.Value(transaction_id, output_type=str),
                 )
             ),
             order_by=[
@@ -222,14 +245,21 @@ async def async_rollback_to_timestamp(timestamp: float) -> None:
             table=glue.SchemaReference(name='Metadata', version=glue.Version.LATEST),
             where=glue.Conditions(
                 glue.Condition(
-                    field=glue.FieldReference(field=glue.Field(name='updated_at'), table_name='Metadata'),
+                    left=glue.FieldReferenceExpression(
+                        field_reference=glue.FieldReference(field=glue.Field(name='updated_at'), table_name='Metadata'),
+                    ),
                     lookup=glue.FieldLookup.GT,
-                    value=glue.Value(timestamp),
+                    right=glue.Value(timestamp),
                 ),
                 glue.Condition(
-                    field=glue.FieldReference(field=glue.Field(name='prior_version'), table_name='Metadata'),
+                    left=glue.FieldReferenceExpression(
+                        field_reference=glue.FieldReference(
+                            field=glue.Field(name='prior_version'),
+                            table_name='Metadata',
+                        ),
+                    ),
                     lookup=glue.FieldLookup.ISNULL,
-                    value=glue.Value(True),
+                    right=glue.Value(True),
                 ),
             ),
         )
@@ -242,14 +272,20 @@ async def async_rollback_to_timestamp(timestamp: float) -> None:
             table=glue.SchemaReference(name='Metadata', version=glue.Version.LATEST),
             where=glue.Conditions(
                 glue.Condition(
-                    field=glue.FieldReference(field=glue.Field(name='updated_at'), table_name='Metadata'),
+                    left=glue.FieldReferenceExpression(
+                        field_reference=glue.FieldReference(field=glue.Field(name='updated_at'), table_name='Metadata'),
+                    ),
                     lookup=glue.FieldLookup.GT,
-                    value=glue.Value(timestamp),
+                    right=glue.Value(timestamp),
                 ),
                 glue.Condition(
-                    field=glue.FieldReference(field=glue.Field(name='prior_version'), table_name='Metadata'),
+                    left=glue.FieldReferenceExpression(
+                        field_reference=glue.FieldReference(
+                            field=glue.Field(name='prior_version'), table_name='Metadata'
+                        ),
+                    ),
                     lookup=glue.FieldLookup.ISNULL,
-                    value=glue.Value(False),
+                    right=glue.Value(False),
                 ),
             ),
         )
@@ -277,9 +313,12 @@ async def async_rollback_to_timestamp(timestamp: float) -> None:
             _parent_field.child.child.parent = _parent_field.child  # type: ignore[union-attr]
             _conditions.append(
                 glue.Condition(
-                    field=glue.FieldReference(field=_parent_field, table_name='Metadata'),
+                    left=glue.FieldReferenceExpression(
+                        field_reference=glue.FieldReference(field=_parent_field, table_name='Metadata'),
+                        output_type=str,
+                    ),
                     lookup=glue.FieldLookup.EQ,
-                    value=glue.Value(transaction_id),
+                    right=glue.Value(transaction_id, output_type=str),
                 )
             )
 
@@ -288,9 +327,14 @@ async def async_rollback_to_timestamp(timestamp: float) -> None:
                 table=glue.SchemaReference(name='Metadata', version=glue.Version.LATEST),
                 where=glue.Conditions(
                     glue.Condition(
-                        field=glue.FieldReference(field=glue.Field(name='updated_at'), table_name='Metadata'),
+                        left=glue.FieldReferenceExpression(
+                            field_reference=glue.FieldReference(
+                                field=glue.Field(name='updated_at'),
+                                table_name='Metadata',
+                            ),
+                        ),
                         lookup=glue.FieldLookup.LTE,
-                        value=glue.Value(timestamp),
+                        right=glue.Value(timestamp),
                     ),
                     glue.Conditions(*_conditions, connector=glue.FilterConnector.OR),
                 ),
@@ -302,9 +346,7 @@ async def async_rollback_to_timestamp(timestamp: float) -> None:
 
     for m in metadatas_to_delete:
         class_name = m.data['class_schema_reference']['ref']['object_id']
-        schema_type = class_manager.resolve_schema_type(class_name)
-
-        model_class = class_manager.import_model_class(class_name, schema_type)
+        model_class = class_manager.import_class(class_name)
         obj = await (
             model_class.objects.filter(_address__object_id=m.data['object_id'])
             .using(LAKEHOUSE_DB_ALIAS)
@@ -317,8 +359,7 @@ async def async_rollback_to_timestamp(timestamp: float) -> None:
             await obj.adelete()
 
     for object_id, class_name in ids_to_revert:
-        schema_type = class_manager.resolve_schema_type(class_name)
-        model_class = class_manager.import_model_class(class_name, schema_type)
+        model_class = class_manager.import_class(class_name)
 
         obj = await (
             model_class.objects.filter(_address__object_id=object_id)
@@ -327,16 +368,17 @@ async def async_rollback_to_timestamp(timestamp: float) -> None:
             .first()
             .aexecute()
         )
-        old_obj = await (
-            model_class.objects.filter(_address__object_id=object_id, _metadata__updated_at__lte=timestamp)
-            .using(LAKEHOUSE_DB_ALIAS)
-            .order_by('-_metadata__updated_at')
-            .first()
-            .aexecute()
-        )
+        old_obj = await obj.aprevious_version()  # type: ignore[union-attr]
+        # old_obj = await (
+        #     model_class.objects.filter(_address__object_id=object_id, _metadata__updated_at__lte=timestamp)
+        #     .using(LAKEHOUSE_DB_ALIAS)
+        #     .order_by('-_metadata__updated_at')
+        #     .first()
+        #     .aexecute()
+        # )
 
         if obj and old_obj:
-            for field, value in old_obj.model_dump().items():
+            for field, value in (await old_obj.amodel_dump()).items():
                 setattr(obj, field, value)
 
             await obj.asave()
@@ -373,9 +415,12 @@ async def async_rollback_transaction(transaction_id: str) -> None:
             table=glue.SchemaReference(name='Metadata', version=glue.Version.LATEST),
             where=glue.Conditions(
                 glue.Condition(
-                    field=glue.FieldReference(field=_parent_field, table_name='Metadata'),
+                    left=glue.FieldReferenceExpression(
+                        field_reference=glue.FieldReference(field=_parent_field, table_name='Metadata'),
+                        output_type=str,
+                    ),
                     lookup=glue.FieldLookup.EQ,
-                    value=glue.Value(transaction_id),
+                    right=glue.Value(transaction_id, output_type=str),
                 )
             ),
             order_by=[

amsdal/utils/rollback/__init__.pyi

@@ -1,3 +1,6 @@
+from amsdal_data.transactions.decorators import async_transaction, transaction
+
+@transaction
 def rollback_to_timestamp(timestamp: float) -> None:
     """
     Rollback the data to the given timestamp
@@ -6,6 +9,7 @@ def rollback_to_timestamp(timestamp: float) -> None:
     Returns:
         None
     """
+@transaction
 def rollback_transaction(transaction_id: str) -> None:
     """
     Rollback the data to the point in time before the given transaction
@@ -14,6 +18,7 @@ def rollback_transaction(transaction_id: str) -> None:
     Returns:
         None
     """
+@async_transaction
 async def async_rollback_to_timestamp(timestamp: float) -> None:
     """
     Rollback the data to the given timestamp
@@ -22,6 +27,7 @@ async def async_rollback_to_timestamp(timestamp: float) -> None:
     Returns:
         None
     """
+@async_transaction
 async def async_rollback_transaction(transaction_id: str) -> None:
     """
     Rollback the data to the point in time before the given transaction

amsdal/utils/tests/enums.py

@@ -13,6 +13,4 @@ class StateOption(str, Enum):
 
 class LakehouseOption(str, Enum):
     postgres = 'postgres'
-    postgres_immutable = 'postgres-immutable'
     sqlite = 'sqlite'
-    sqlite_immutable = 'sqlite-immutable'