amsdal 0.5.34__cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

amsdal/services/external_model_generator.py

@@ -0,0 +1,350 @@
+"""
+External Model Generator Service.
+
+Generates ExternalModel classes from external connection schemas.
+This allows runtime model generation from external databases without
+manual model definition.
+"""
+
+from typing import Any
+from typing import cast
+
+from amsdal_data.connections.external.base import SchemaIntrospectionProtocol
+from amsdal_models.classes.external_model import ExternalModel
+from amsdal_models.utils.schema_converter import ExternalSchemaConverter
+from amsdal_utils.schemas.schema import ObjectSchema
+
+from amsdal.services.external_connections import ExternalConnectionManager
+
+
+class ExternalModelGenerator:
+    """
+    Service for generating ExternalModel classes from external connections.
+
+    This service introspects external database schemas and generates
+    corresponding ExternalModel classes that can be used immediately
+    for querying the external data.
+
+    Features:
+    - Automatic schema introspection
+    - Type mapping (SQL types -> Python types)
+    - Primary key detection
+    - In-memory model class generation
+    - No lakehouse schema creation
+
+    Example usage:
+        # Generate models for a single table
+        generator = ExternalModelGenerator()
+        User = generator.generate_model('external_db', 'users')
+
+        # Now use the generated model
+        users = User.objects.filter(active=True).execute()
+
+        # Generate models for all tables
+        models = generator.generate_models_for_connection('external_db')
+        User = models['User']
+        Post = models['Post']
+    """
+
+    def __init__(self) -> None:
+        self._connection_manager = ExternalConnectionManager()
+        self._schema_converter = ExternalSchemaConverter()
+
+    def generate_model(
+        self,
+        connection_name: str,
+        table_name: str,
+        model_name: str | None = None,
+    ) -> type[ExternalModel]:
+        """
+        Generate an ExternalModel class for a specific table.
+
+        Args:
+            connection_name: Name of the external connection
+            table_name: Name of the table to generate model for
+            model_name: Optional custom model name (defaults to classified table name)
+
+        Returns:
+            type[ExternalModel]: Generated model class ready to use
+
+        Raises:
+            ValueError: If connection doesn't support schema introspection
+            ConnectionError: If connection is not available
+            RuntimeError: If model generation fails
+
+        Example:
+            generator = ExternalModelGenerator()
+            User = generator.generate_model('external_db', 'users')
+
+            # Query using the generated model
+            active_users = User.objects.filter(active=True).execute()
+        """
+        # Get the connection
+        connection = self._connection_manager.get_connection(connection_name)
+
+        # Check if connection supports schema introspection
+        if not isinstance(connection, SchemaIntrospectionProtocol):  # type: ignore[misc]
+            msg = (
+                f"Connection '{connection_name}' does not support schema introspection. "
+                f'Connection type: {type(connection).__name__}'
+            )
+            raise ValueError(msg)
+
+        # Get table schema
+        table_schema = connection.get_table_schema(table_name)
+
+        # Convert to ObjectSchema
+        # Detect connection type and use appropriate converter
+        object_schema = self._convert_schema(
+            connection=connection,
+            table_name=table_name,
+            table_schema=table_schema,
+            connection_name=connection_name,
+        )
+
+        # Generate model class from ObjectSchema
+        model_class = self._create_model_class(object_schema, model_name)
+
+        return model_class
+
+    def generate_models_for_connection(
+        self,
+        connection_name: str,
+        table_names: list[str] | None = None,
+    ) -> dict[str, type[ExternalModel]]:
+        """
+        Generate ExternalModel classes for all tables in a connection.
+
+        Args:
+            connection_name: Name of the external connection
+            table_names: Optional list of specific tables to generate models for.
+                        If None, generates models for all tables.
+
+        Returns:
+            dict[str, type[ExternalModel]]: Dictionary mapping model names to model classes
+
+        Raises:
+            ValueError: If connection doesn't support schema introspection
+            ConnectionError: If connection is not available
+
+        Example:
+            generator = ExternalModelGenerator()
+            models = generator.generate_models_for_connection('external_db')
+
+            # Access generated models
+            User = models['User']
+            Post = models['Post']
+            Comment = models['Comment']
+
+            # Or generate only specific tables
+            models = generator.generate_models_for_connection(
+                'external_db',
+                table_names=['users', 'posts']
+            )
+        """
+        # Get the connection
+        connection = self._connection_manager.get_connection(connection_name)
+
+        # Check if connection supports schema introspection
+        if not isinstance(connection, SchemaIntrospectionProtocol):  # type: ignore[misc]
+            msg = (
+                f"Connection '{connection_name}' does not support schema introspection. "
+                f'Connection type: {type(connection).__name__}'
+            )
+            raise ValueError(msg)
+
+        # Get list of tables
+        if table_names is None:
+            table_names = connection.get_table_names()
+
+        # Generate models for each table
+        models: dict[str, type[ExternalModel]] = {}
+        for table_name in table_names:
+            try:
+                model = self.generate_model(connection_name, table_name)
+                models[model.__name__] = model
+            except Exception as e:
+                # Log error but continue with other tables
+                print(f"Warning: Failed to generate model for table '{table_name}': {e}")
+                continue
+
+        return models
+
+    def _convert_schema(
+        self,
+        connection: Any,
+        table_name: str,
+        table_schema: list[dict[str, Any]],
+        connection_name: str,
+    ) -> ObjectSchema:
+        """
+        Convert raw table schema to ObjectSchema based on connection type.
+
+        Args:
+            connection: The connection object
+            table_name: Name of the table
+            table_schema: Raw schema data from connection
+            connection_name: Name of the connection
+
+        Returns:
+            ObjectSchema: Converted schema
+        """
+        # Detect connection type and use appropriate converter
+        connection_type = type(connection).__name__
+
+        if 'sqlite' in connection_type.lower():
+            return self._schema_converter.sqlite_schema_to_object_schema(
+                table_name=table_name,
+                columns=table_schema,
+                connection_name=connection_name,
+            )
+
+        # For other connection types, try to use generic converter
+        # First, try to detect the schema format
+        if table_schema and isinstance(table_schema[0], dict):
+            # Check if it's SQLite format (has 'cid', 'name', 'type', 'pk', etc.)
+            if all(key in table_schema[0] for key in ('cid', 'name', 'type')):
+                return self._schema_converter.sqlite_schema_to_object_schema(
+                    table_name=table_name,
+                    columns=table_schema,
+                    connection_name=connection_name,
+                )
+
+            # Check if it's PostgreSQL format (has 'column_name', 'data_type', etc.)
+            if 'column_name' in table_schema[0] and 'data_type' in table_schema[0]:
+                return self._schema_converter.postgres_schema_to_object_schema(
+                    table_name=table_name,
+                    columns=table_schema,
+                    connection_name=connection_name,
+                )
+
+            # Try generic converter with format normalization
+            normalized_columns = self._normalize_schema_format(table_schema)
+            return self._schema_converter.generic_schema_to_object_schema(
+                table_name=table_name,
+                columns=normalized_columns,
+                connection_name=connection_name,
+            )
+
+        msg = f'Unknown schema format for connection type: {connection_type}'
+        raise ValueError(msg)
+
+    def _normalize_schema_format(self, table_schema: list[dict[str, Any]]) -> list[dict[str, Any]]:
+        """
+        Normalize various schema formats to generic format.
+
+        Converts various schema formats to the format expected by generic_schema_to_object_schema:
+        {'name': str, 'type': str, 'nullable': bool, 'primary_key': bool, 'default': Any}
+        """
+        normalized = []
+
+        for column in table_schema:
+            # Try to extract name
+            name = column.get('name') or column.get('column_name') or column.get('field')
+
+            # Try to extract type
+            col_type = column.get('type') or column.get('data_type') or column.get('field_type') or 'TEXT'
+
+            # Try to extract nullable
+            nullable = True
+            if 'nullable' in column:
+                nullable = column['nullable']
+            elif 'is_nullable' in column:
+                nullable = column['is_nullable'] in (True, 'YES', 'yes', 1)
+            elif 'notnull' in column:
+                nullable = column['notnull'] in (False, 0)
+
+            # Try to extract primary key
+            pk = column.get('primary_key') or column.get('pk') or False
+            if isinstance(pk, int):
+                pk = pk > 0
+
+            # Try to extract default
+            default = column.get('default') or column.get('dflt_value') or column.get('column_default')
+
+            normalized.append(
+                {
+                    'name': name,
+                    'type': col_type,
+                    'nullable': nullable,
+                    'primary_key': pk,
+                    'default': default,
+                }
+            )
+
+        return normalized
+
+    def _create_model_class(
+        self,
+        object_schema: ObjectSchema,
+        custom_name: str | None = None,
+    ) -> type[ExternalModel]:
+        """
+        Create an ExternalModel class from ObjectSchema.
+
+        Args:
+            object_schema: The schema to create model from
+            custom_name: Optional custom model name
+
+        Returns:
+            type[ExternalModel]: Generated model class
+        """
+        # Extract model metadata from schema
+        model_name = custom_name or object_schema.title
+        table_name = cast(str, object_schema.__table_name__)  # type: ignore[attr-defined]
+        connection_name = cast(str, object_schema.__connection__)  # type: ignore[attr-defined]
+        pk_fields = getattr(object_schema, '__primary_key__', None)
+
+        # Build class attributes
+        class_attrs: dict[str, Any] = {
+            '__table_name__': table_name,
+            '__connection__': connection_name,
+            '__module__': __name__,
+        }
+
+        # Add primary key if present
+        if pk_fields:
+            # For composite keys, use list; for single key, use string
+            if len(pk_fields) == 1:
+                class_attrs['__primary_key__'] = pk_fields[0]
+            else:
+                class_attrs['__primary_key__'] = pk_fields
+
+        # Add field annotations from schema properties
+        annotations: dict[str, type] = {}
+        if object_schema.properties:
+            for field_name, field_def in object_schema.properties.items():
+                # Map CoreTypes to Python types for annotations
+                field_type = self._core_type_to_python_type(getattr(field_def, 'type', 'string'))
+                annotations[field_name] = field_type
+
+        class_attrs['__annotations__'] = annotations
+
+        # Create the model class dynamically
+        model_class = type(model_name, (ExternalModel,), class_attrs)
+
+        return cast(type[ExternalModel], model_class)
+
+    @staticmethod
+    def _core_type_to_python_type(core_type: str) -> type:
+        """
+        Convert CoreType string to Python type for annotations.
+
+        Args:
+            core_type: CoreType value (e.g., 'string', 'integer')
+
+        Returns:
+            type: Corresponding Python type
+        """
+        type_mapping = {
+            'string': str,
+            'integer': int,
+            'number': float,
+            'boolean': bool,
+            'date': str,  # Will be string representation
+            'datetime': str,  # Will be string representation
+            'binary': bytes,
+            'array': list,
+            'dictionary': dict,
+        }
+        return type_mapping.get(core_type, str)

amsdal/services/external_model_generator.pyi

@@ -0,0 +1,134 @@
+from _typeshed import Incomplete
+from amsdal.services.external_connections import ExternalConnectionManager as ExternalConnectionManager
+from amsdal_models.classes.external_model import ExternalModel
+from amsdal_utils.schemas.schema import ObjectSchema as ObjectSchema
+from typing import Any
+
+class ExternalModelGenerator:
+    """
+    Service for generating ExternalModel classes from external connections.
+
+    This service introspects external database schemas and generates
+    corresponding ExternalModel classes that can be used immediately
+    for querying the external data.
+
+    Features:
+    - Automatic schema introspection
+    - Type mapping (SQL types -> Python types)
+    - Primary key detection
+    - In-memory model class generation
+    - No lakehouse schema creation
+
+    Example usage:
+        # Generate models for a single table
+        generator = ExternalModelGenerator()
+        User = generator.generate_model('external_db', 'users')
+
+        # Now use the generated model
+        users = User.objects.filter(active=True).execute()
+
+        # Generate models for all tables
+        models = generator.generate_models_for_connection('external_db')
+        User = models['User']
+        Post = models['Post']
+    """
+    _connection_manager: Incomplete
+    _schema_converter: Incomplete
+    def __init__(self) -> None: ...
+    def generate_model(self, connection_name: str, table_name: str, model_name: str | None = None) -> type[ExternalModel]:
+        """
+        Generate an ExternalModel class for a specific table.
+
+        Args:
+            connection_name: Name of the external connection
+            table_name: Name of the table to generate model for
+            model_name: Optional custom model name (defaults to classified table name)
+
+        Returns:
+            type[ExternalModel]: Generated model class ready to use
+
+        Raises:
+            ValueError: If connection doesn't support schema introspection
+            ConnectionError: If connection is not available
+            RuntimeError: If model generation fails
+
+        Example:
+            generator = ExternalModelGenerator()
+            User = generator.generate_model('external_db', 'users')
+
+            # Query using the generated model
+            active_users = User.objects.filter(active=True).execute()
+        """
+    def generate_models_for_connection(self, connection_name: str, table_names: list[str] | None = None) -> dict[str, type[ExternalModel]]:
+        """
+        Generate ExternalModel classes for all tables in a connection.
+
+        Args:
+            connection_name: Name of the external connection
+            table_names: Optional list of specific tables to generate models for.
+                        If None, generates models for all tables.
+
+        Returns:
+            dict[str, type[ExternalModel]]: Dictionary mapping model names to model classes
+
+        Raises:
+            ValueError: If connection doesn't support schema introspection
+            ConnectionError: If connection is not available
+
+        Example:
+            generator = ExternalModelGenerator()
+            models = generator.generate_models_for_connection('external_db')
+
+            # Access generated models
+            User = models['User']
+            Post = models['Post']
+            Comment = models['Comment']
+
+            # Or generate only specific tables
+            models = generator.generate_models_for_connection(
+                'external_db',
+                table_names=['users', 'posts']
+            )
+        """
+    def _convert_schema(self, connection: Any, table_name: str, table_schema: list[dict[str, Any]], connection_name: str) -> ObjectSchema:
+        """
+        Convert raw table schema to ObjectSchema based on connection type.
+
+        Args:
+            connection: The connection object
+            table_name: Name of the table
+            table_schema: Raw schema data from connection
+            connection_name: Name of the connection
+
+        Returns:
+            ObjectSchema: Converted schema
+        """
+    def _normalize_schema_format(self, table_schema: list[dict[str, Any]]) -> list[dict[str, Any]]:
+        """
+        Normalize various schema formats to generic format.
+
+        Converts various schema formats to the format expected by generic_schema_to_object_schema:
+        {'name': str, 'type': str, 'nullable': bool, 'primary_key': bool, 'default': Any}
+        """
+    def _create_model_class(self, object_schema: ObjectSchema, custom_name: str | None = None) -> type[ExternalModel]:
+        """
+        Create an ExternalModel class from ObjectSchema.
+
+        Args:
+            object_schema: The schema to create model from
+            custom_name: Optional custom model name
+
+        Returns:
+            type[ExternalModel]: Generated model class
+        """
+    @staticmethod
+    def _core_type_to_python_type(core_type: str) -> type:
+        """
+        Convert CoreType string to Python type for annotations.
+
+        Args:
+            core_type: CoreType value (e.g., 'string', 'integer')
+
+        Returns:
+            type: Corresponding Python type
+        """

amsdal/services/transaction_execution.pyi

@@ -0,0 +1,93 @@
+import ast
+from _typeshed import Incomplete
+from amsdal.configs.main import settings as settings
+from amsdal.errors import TransactionNotFoundError as TransactionNotFoundError
+from amsdal_utils.utils.singleton import Singleton
+from collections.abc import Callable as Callable, Generator
+from pathlib import Path
+from typing import Any
+
+logger: Incomplete
+
+def is_transaction(statement: ast.AST) -> bool:
+    """
+    Determines if a given AST statement is a transaction function.
+
+    This function checks if the provided AST statement is an asynchronous or synchronous function
+    definition that is decorated with the `transaction` decorator.
+
+    Args:
+        statement (ast.AST): The AST statement to check.
+
+    Returns:
+        bool: True if the statement is a transaction function, False otherwise.
+    """
+def annotation_is_model(annotation: Any) -> bool: ...
+
+class TransactionExecutionService(metaclass=Singleton):
+    """
+    Service for executing transactions.
+
+    This class provides methods to execute transactions, load transaction functions,
+    and handle asynchronous transactions. It ensures that transactions are executed
+    with the correct arguments and handles any necessary preprocessing of arguments.
+    """
+    _transactions: dict[str, Callable[..., Any]]
+    def __init__(self) -> None: ...
+    def execute_transaction(self, transaction_name: str, args: dict[str, Any], *, load_references: bool = True) -> Any:
+        """
+        Executes a transaction with the given name and arguments.
+
+        This method retrieves the transaction function by its name, processes the arguments,
+        and executes the transaction. It handles both synchronous and asynchronous transactions
+        and performs necessary preprocessing of arguments, such as loading references.
+
+        Args:
+            transaction_name (str): The name of the transaction to execute.
+            args (dict[str, Any]): The arguments to pass to the transaction function.
+            load_references (bool, optional): Whether to load references in the arguments. Defaults to True.
+
+        Returns:
+            Any: The result of the transaction execution.
+        """
+    async def async_execute_transaction(self, transaction_name: str, args: dict[str, Any], *, load_references: bool = True) -> Any:
+        """
+        Executes a transaction with the given name and arguments.
+
+        This method retrieves the transaction function by its name, processes the arguments,
+        and executes the transaction. It handles both synchronous and asynchronous transactions
+        and performs necessary preprocessing of arguments, such as loading references.
+
+        Args:
+            transaction_name (str): The name of the transaction to execute.
+            args (dict[str, Any]): The arguments to pass to the transaction function.
+            load_references (bool, optional): Whether to load references in the arguments. Defaults to True.
+
+        Returns:
+            Any: The result of the transaction execution.
+        """
+    def get_transaction_func(self, transaction_name: str) -> Callable[..., Any]:
+        """
+        Retrieves the transaction function by its name.
+
+        This method checks if the transaction function is already loaded in the `_transactions` dictionary.
+        If not, it attempts to load the transaction function from the available transaction definitions.
+
+        Args:
+            transaction_name (str): The name of the transaction function to retrieve.
+
+        Returns:
+            Callable[..., Any]: The transaction function corresponding to the given name.
+
+        Raises:
+            TransactionNotFoundError: If the transaction function with the specified name is not found.
+        """
+    @staticmethod
+    def _run_async_transaction(transaction_func: Callable[..., Any], args: dict[str, Any]) -> Any: ...
+    def _load_transaction(self, transaction_name: str) -> Callable[..., Any]: ...
+    @classmethod
+    def _get_transaction_definitions(cls) -> Generator[tuple[ast.FunctionDef | ast.AsyncFunctionDef, Path], None, None]: ...
+    @classmethod
+    def _iterate_module(cls, module_path: Path) -> Generator[tuple[ast.FunctionDef | ast.AsyncFunctionDef, Path], None, None]: ...
+    @classmethod
+    def _iterate_file(cls, file_path: Path) -> Generator[tuple[ast.FunctionDef | ast.AsyncFunctionDef, Path], None, None]: ...

amsdal/storages/__init__.py

@@ -0,0 +1,20 @@
+from amsdal_models.storage.base import Storage
+
+from amsdal.configs.main import settings
+
+_DEFAULT_STORAGE = None
+
+
+def set_default_storage(storage: Storage) -> None:
+    global _DEFAULT_STORAGE  # noqa: PLW0603
+    _DEFAULT_STORAGE = storage
+
+
+def default_storage() -> Storage:
+    global _DEFAULT_STORAGE  # noqa: PLW0603
+
+    if _DEFAULT_STORAGE is None:
+        # Determine backend from settings
+        class_path = settings.DEFAULT_FILE_STORAGE
+        _DEFAULT_STORAGE = Storage.from_storage_spec({'storage_class': class_path})
+    return _DEFAULT_STORAGE

amsdal/storages/__init__.pyi

@@ -0,0 +1,8 @@
+from _typeshed import Incomplete
+from amsdal.configs.main import settings as settings
+from amsdal_models.storage.base import Storage
+
+_DEFAULT_STORAGE: Incomplete
+
+def set_default_storage(storage: Storage) -> None: ...
+def default_storage() -> Storage: ...