dara-core 1.19.0__py3-none-any.whl → 1.20.0a1__py3-none-any.whl

dara/core/interactivity/filtering.py

@@ -20,10 +20,11 @@ from __future__ import annotations
 import re
 from datetime import datetime, timezone
 from enum import Enum
-from typing import Any, List, Optional, Tuple, Union, cast
+from typing import Any, List, Optional, Tuple, Union, cast, overload
 
 import numpy
-from pandas import DataFrame, Series  # noqa: F401
+from pandas import DataFrame, Series
+from pydantic import field_validator  # noqa: F401
 
 from dara.core.base_definitions import DaraBaseModel as BaseModel
 from dara.core.logging import dev_logger
@@ -31,6 +32,13 @@ from dara.core.logging import dev_logger
 COLUMN_PREFIX_REGEX = re.compile(r'__(?:col|index)__\d+__')
 
 
+def clean_column_name(col: str) -> str:
+    """
+    Cleans a column name by removing the index or col prefix
+    """
+    return re.sub(COLUMN_PREFIX_REGEX, '', col)
+
+
 class Pagination(BaseModel):
     """
     Model representing pagination to be applied to a dataset.
@@ -44,6 +52,13 @@ class Pagination(BaseModel):
     orderBy: Optional[str] = None
     index: Optional[str] = None
 
+    @field_validator('orderBy', mode='before')
+    @classmethod
+    def clean_order_by(cls, order_by):
+        if order_by is None:
+            return None
+        return clean_column_name(order_by)
+
 
 class QueryCombinator(str, Enum):
     AND = 'AND'
@@ -231,6 +246,18 @@ def _resolve_filter_query(data: DataFrame, query: FilterQuery) -> Optional[Serie
         raise ValueError(f'Unknown query type {type(query)}')
 
 
+@overload
+def apply_filters(
+    data: DataFrame, filters: Optional[FilterQuery] = None, pagination: Optional[Pagination] = None
+) -> Tuple[DataFrame, int]: ...
+
+
+@overload
+def apply_filters(
+    data: None, filters: Optional[FilterQuery] = None, pagination: Optional[Pagination] = None
+) -> Tuple[None, int]: ...
+
+
 def apply_filters(
     data: Optional[DataFrame], filters: Optional[FilterQuery] = None, pagination: Optional[Pagination] = None
 ) -> Tuple[Optional[DataFrame], int]:

dara/core/interactivity/loop_variable.py

@@ -2,10 +2,10 @@ from typing import List, Optional
 
 from pydantic import Field, SerializerFunctionWrapHandler, model_serializer
 
-from .non_data_variable import NonDataVariable
+from .client_variable import ClientVariable
 
 
-class LoopVariable(NonDataVariable):
+class LoopVariable(ClientVariable):
     """
     A LoopVariable is a type of variable that represents an item in a list.
     It should be constructed using a parent Variable's `.list_item` property.

dara/core/interactivity/non_data_variable.py

@@ -1,71 +1,8 @@
-"""
-Copyright 2023 Impulse Innovations Limited
-
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
+from typing_extensions import TypeAlias
 
-    http://www.apache.org/licenses/LICENSE-2.0
+from .client_variable import *  # noqa: F403
 
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
+NonDataVariable: TypeAlias = ClientVariable  # noqa: F405
+"""
+Deprecated alias for ClientVariable
 """
-
-from __future__ import annotations
-
-import abc
-from typing import Optional
-
-from dara.core.interactivity.any_variable import AnyVariable
-
-
-class NonDataVariable(AnyVariable, abc.ABC):
-    """
-    NonDataVariable represents any variable that is not specifically designed to hold datasets (i.e. Variable, DerivedVariable, UrlVariable)
-
-    :param uid: the unique identifier for this variable; if not provided a random one is generated
-    """
-
-    uid: str
-
-    def __init__(self, uid: Optional[str] = None, **kwargs) -> None:
-        super().__init__(uid=uid, **kwargs)
-
-    @property
-    def list_item(self):
-        """
-        Get a LoopVariable that represents the current item in the list.
-        Should only be used in conjunction with the `For` component.
-
-        Note that it is a type of a Variable so it can be used in places where a regular Variable is expected.
-
-        By default, the entire list item is used as the item.
-
-        `LoopVariable` supports nested property access using `get` or index access i.e. `[]`.
-        You can mix and match those two methods to access nested properties as they are equivalent.
-
-        ```python
-        my_list = Variable(['foo', 'bar', 'baz'])
-
-        # Represents the entire item in the list
-        my_list.list_item
-
-        my_list_of_objects = Variable([
-            {'id': 1, 'name': 'John', 'data': {'city': 'London', 'country': 'UK'}},
-            {'id': 2, 'name': 'Jane', 'data': {'city': 'Paris', 'country': 'France'}},
-        ])
-
-        # Represents the item 'name' property
-        my_list_of_objects.list_item['name']
-
-        # Represents the item 'data.country' property
-        my_list_of_objects.list_item.get('data')['country']
-        """
-
-        from .loop_variable import LoopVariable
-
-        return LoopVariable()

dara/core/interactivity/plain_variable.py

@@ -17,9 +17,10 @@ limitations under the License.
 
 from __future__ import annotations
 
+import warnings
 from contextlib import contextmanager
 from contextvars import ContextVar
-from typing import Any, Callable, Generic, List, Optional, TypeVar
+from typing import Any, Callable, Dict, Generic, List, Optional, TypeVar, Union
 
 from fastapi.encoders import jsonable_encoder
 from pydantic import (
@@ -30,12 +31,11 @@ from pydantic import (
     model_serializer,
 )
 
-from dara.core.interactivity.derived_data_variable import DerivedDataVariable
+from dara.core.interactivity.client_variable import ClientVariable
 from dara.core.interactivity.derived_variable import DerivedVariable
-from dara.core.interactivity.non_data_variable import NonDataVariable
 from dara.core.internal.utils import call_async
 from dara.core.logging import dev_logger
-from dara.core.persistence import PersistenceStore
+from dara.core.persistence import BackendStore, BrowserStore, PersistenceStore
 
 VARIABLE_INIT_OVERRIDE = ContextVar[Optional[Callable[[dict], dict]]]('VARIABLE_INIT_OVERRIDE', default=None)
 
@@ -44,13 +44,12 @@ PersistenceStoreType_co = TypeVar('PersistenceStoreType_co', bound=PersistenceSt
 
 
 # TODO: once Python supports a default value for a generic type properly we can make PersistenceStoreType a second generic param
-class Variable(NonDataVariable, Generic[VariableType]):
+class Variable(ClientVariable, Generic[VariableType]):
     """
     A Variable represents a dynamic value in the system that can be read and written to by components and actions
     """
 
     default: Optional[VariableType] = None
-    persist_value: bool = False
     store: Optional[PersistenceStore] = None
     uid: str
     nested: List[str] = Field(default_factory=list)
@@ -63,6 +62,7 @@ class Variable(NonDataVariable, Generic[VariableType]):
         uid: Optional[str] = None,
         store: Optional[PersistenceStoreType_co] = None,
         nested: Optional[List[str]] = None,
+        **kwargs,
     ):
         """
         A Variable represents a dynamic value in the system that can be read and written to by components and actions
@@ -74,17 +74,31 @@ class Variable(NonDataVariable, Generic[VariableType]):
         """
         if nested is None:
             nested = []
-        kwargs = {'default': default, 'persist_value': persist_value, 'uid': uid, 'store': store, 'nested': nested}
+        kwargs = {
+            'default': default,
+            'uid': uid,
+            'store': store,
+            'nested': nested,
+            **kwargs,
+        }
 
         # If an override is active, run the kwargs through it
         override = VARIABLE_INIT_OVERRIDE.get()
         if override is not None:
             kwargs = override(kwargs)
 
-        if kwargs.get('store') is not None and kwargs.get('persist_value'):
+        if kwargs.get('store') is not None and persist_value:
             # TODO: this is temporary, persist_value will eventually become a type of store
             raise ValueError('Cannot provide a Variable with both a store and persist_value set to True')
 
+        if persist_value:
+            warnings.warn(
+                '`persist_value` is deprecated and will be removed in a future version. Use `store=dara.core.persistence.BrowserStore(...)` instead.',
+                DeprecationWarning,
+                stacklevel=2,
+            )
+            kwargs['store'] = BrowserStore()
+
         super().__init__(**kwargs)  # type: ignore
 
         if self.store:
@@ -116,7 +130,7 @@ class Variable(NonDataVariable, Generic[VariableType]):
         Override the init function of all Variables created within the context of this function.
 
         ```python
-        with Variable.init_override(lambda kwargs: {**kwargs, 'persist_value': True}):
+        with Variable.init_override(lambda kwargs: {**kwargs, 'store': ...}):
             var = Variable()
         ```
 
@@ -253,13 +267,72 @@ class Variable(NonDataVariable, Generic[VariableType]):
 
         :param default: the initial value for the variable, defaults to None
         """
-        if isinstance(other, DerivedDataVariable):
-            raise ValueError(
-                'Cannot create a Variable from a DerivedDataVariable, only standard DerivedVariables are allowed'
-            )
-
         return cls(default=other)  # type: ignore
 
+    async def write(self, value: Any, notify=True, ignore_channel: Optional[str] = None):
+        """
+        Persist a value to the variable's BackendStore.
+        Raises an error if the variable does not have a BackendStore attached.
+
+        If scope='user', the value is written for the current user so the method can only
+        be used in authenticated contexts.
+
+        :param value: value to write
+        :param notify: whether to broadcast the new value to clients
+        :param ignore_channel: if passed, ignore the specified websocket channel when broadcasting
+        """
+        assert isinstance(self.store, BackendStore), 'This method can only be used with a BackendStore'
+        return await self.store.write(value, notify=notify, ignore_channel=ignore_channel)
+
+    async def write_partial(self, data: Union[List[Dict[str, Any]], Any], notify: bool = True):
+        """
+        Apply partial updates to the variable's BackendStore using JSON Patch operations or automatic diffing.
+        Raises an error if the variable does not have a BackendStore attached.
+
+        If scope='user', the patches are applied for the current user so the method can only
+        be used in authenticated contexts.
+
+        :param data: Either a list of JSON patch operations (RFC 6902) or a full object to diff against current value
+        :param notify: whether to broadcast the patches to clients
+        """
+        assert isinstance(self.store, BackendStore), 'This method can only be used with a BackendStore'
+        return await self.store.write_partial(data, notify=notify)
+
+    async def read(self):
+        """
+        Read a value from the variable's BackendStore.
+        Raises an error if the variable does not have a BackendStore attached.
+
+        If scope='user', the value is read for the current user so the method can only
+        be used in authenticated contexts.
+        """
+        assert isinstance(self.store, BackendStore), 'This method can only be used with a BackendStore'
+        return await self.store.read()
+
+    async def delete(self, notify=True):
+        """
+        Delete the persisted value from the variable's BackendStore.
+        Raises an error if the variable does not have a BackendStore attached.
+
+        If scope='user', the value is deleted for the current user so the method can only
+        be used in authenticated contexts.
+
+        :param notify: whether to broadcast that the value was deleted to clients
+        """
+        assert isinstance(self.store, BackendStore), 'This method can only be used with a BackendStore'
+        return await self.store.delete(notify=notify)
+
+    async def get_all(self) -> Dict[str, Any]:
+        """
+        Get all the values from the variable's BackendStore as a dictionary of key-value pairs.
+        Raises an error if the variable does not have a BackendStore attached.
+
+        For global scope, the dictionary contains a single key-value pair `{'global': value}`.
+        For user scope, the dictionary contains a key-value pair for each user `{'user1': value1, 'user2': value2, ...}`.
+        """
+        assert isinstance(self.store, BackendStore), 'This method can only be used with a BackendStore'
+        return await self.store.get_all()
+
     @model_serializer(mode='wrap')
     def ser_model(self, nxt: SerializerFunctionWrapHandler) -> dict:
         parent_dict = nxt(self)

dara/core/interactivity/server_variable.py

@@ -0,0 +1,325 @@
+import abc
+from collections import defaultdict
+from typing import Any, DefaultDict, Dict, Literal, Optional, Tuple, Union
+
+from pandas import DataFrame
+from pydantic import BaseModel, ConfigDict, Field, SerializerFunctionWrapHandler, model_serializer
+
+from dara.core.auth.definitions import USER
+from dara.core.base_definitions import CachedRegistryEntry, NonTabularDataError
+from dara.core.interactivity.filtering import FilterQuery, Pagination, apply_filters, coerce_to_filter_query
+from dara.core.internal.pandas_utils import DataResponse, append_index, build_data_response
+from dara.core.internal.utils import call_async
+from dara.core.internal.websocket import ServerMessagePayload, WebsocketManager
+
+from .any_variable import AnyVariable
+
+
+class ServerVariableMessage(ServerMessagePayload):
+    typ: Literal['ServerVariable'] = Field(alias='__type', default='ServerVariable')
+    uid: str
+    sequence_number: int
+
+
+class ServerBackend(BaseModel, abc.ABC):
+    scope: Literal['global', 'user']
+
+    @abc.abstractmethod
+    async def write(self, key: str, value: Any):
+        """
+        Persist a value
+        """
+
+    @abc.abstractmethod
+    async def read(self, key: str) -> Any:
+        """
+        Read a value
+        """
+
+    @abc.abstractmethod
+    async def read_filtered(
+        self, key: str, filters: Optional[Union[FilterQuery, dict]] = None, pagination: Optional[Pagination] = None
+    ) -> Tuple[Optional[DataFrame], int]:
+        """
+        Read a value
+        :param filters: filters to apply
+        :param pagination: pagination to apply
+        """
+
+    @abc.abstractmethod
+    async def get_sequence_number(self, key: str) -> int:
+        """
+        Get the sequence number for a given key
+        """
+
+
+class MemoryBackend(ServerBackend):
+    data: Dict[str, Any] = Field(default_factory=dict)
+    sequence_number: DefaultDict[str, int] = Field(default_factory=lambda: defaultdict(int))
+
+    def __init__(self, scope: Literal['user', 'global'] = 'user'):
+        super().__init__(scope=scope)
+
+    async def write(self, key: str, value: Any):
+        self.data[key] = value
+        self.sequence_number[key] += 1
+        return value
+
+    async def read(self, key: str) -> Any:
+        return self.data.get(key)
+
+    async def read_filtered(
+        self, key: str, filters: Optional[Union[FilterQuery, dict]] = None, pagination: Optional[Pagination] = None
+    ) -> Tuple[Optional[DataFrame], int]:
+        dataset = self.data.get(key)
+
+        # print user-friendly error message if the data is not a DataFrame
+        # most likely due to user passing a non-tabular server variable to e.g. a Table
+        if dataset is not None and not isinstance(dataset, DataFrame):
+            raise NonTabularDataError(
+                f'Failed to retrieve ServerVariable tabular data, expected pandas.DataFrame, got {type(dataset)}'
+            )
+
+        dataset = append_index(dataset)
+        return apply_filters(dataset, coerce_to_filter_query(filters), pagination)
+
+    async def get_sequence_number(self, key: str) -> int:
+        return self.sequence_number[key]
+
+
+class ServerVariable(AnyVariable):
+    """
+    A ServerVariable represents server-side data that is synchronized with the client.
+
+    Unlike Variables with BackendStore (which are client state persisted on server),
+    ServerVariable holds data that originates and is managed on the server, with
+    clients receiving reactive updates when the data changes.
+
+    ServerVariable can store any Python object, including non-serializable data like
+    database connections, ML models, or complex objects.
+
+    However, when used with components expecting tabular data (like Table), the data must be
+    serializable or a NonTabularDataError will be raised. The default backend implementation
+    expects the tabular data to be a pandas DataFrame. To support other data types, you can
+    implement a custom backend that translates the data into a filtered DataFrame in the
+    `read_filtered` method.
+
+    ```python
+    import pandas as pd
+    from dara.core import ServerVariable, action
+    from dara.core.interactivity.server_variable import ServerBackend
+    from sklearn.ensemble import RandomForestClassifier
+
+    # Basic usage with DataFrame
+    data = ServerVariable(pd.DataFrame({'a': [1, 2, 3]}))
+
+    # Non-serializable data (ML model)
+    model = ServerVariable(trained_sklearn_model, scope='global')
+
+    # Custom backend
+    class DatabaseBackend(ServerBackend):
+        # ... implements all the methods as DB operations
+
+    data = ServerVariable(backend=DatabaseBackend(...))
+
+    # User-specific data
+    user_prefs = ServerVariable(scope='user')
+
+    @action
+    async def on_click(ctx):
+        # write to the data for the user who initiated the action
+        await user_prefs.write('dark')
+    ```
+
+    :param default: Initial value for the variable (global scope only)
+    :param backend: Custom backend for data storage and retrieval
+    :param scope: 'global' (shared across all users) or 'user' (per-user data)
+    """
+
+    backend: ServerBackend = Field(exclude=True)
+    scope: Literal['user', 'global']
+
+    def __init__(
+        self,
+        default: Optional[Any] = None,
+        backend: Optional[ServerBackend] = None,
+        scope: Literal['user', 'global'] = 'global',
+        uid: Optional[str] = None,
+        **kwargs,
+    ) -> None:
+        from dara.core.internal.registries import server_variable_registry
+
+        if backend is None:
+            backend = MemoryBackend(scope=scope)
+
+        if default is not None:
+            assert scope == 'global', (
+                'ServerVariable can only be used with global scope, cannot initialize user-specific values'
+            )
+            call_async(backend.write, 'global', default)
+
+        super().__init__(uid=uid, backend=backend, scope=scope, **kwargs)
+
+        var_entry = ServerVariableRegistryEntry(uid=str(self.uid), backend=backend)
+        server_variable_registry.register(str(self.uid), var_entry)
+
+    @classmethod
+    async def get_value(cls, entry: 'ServerVariableRegistryEntry'):
+        """
+        Internal method to get the value of a server variable based in its registry entry.
+        """
+        key = cls.get_key(entry.backend.scope)
+        return await entry.backend.read(key)
+
+    @classmethod
+    async def write_value(cls, entry: 'ServerVariableRegistryEntry', value: Any):
+        """
+        Internal method to write the value of a server variable based in its registry entry.
+        """
+        key = cls.get_key(entry.backend.scope)
+        await entry.backend.write(key, value)
+        await cls._notify(entry.uid, key, entry.backend)
+
+    @classmethod
+    async def get_sequence_number(cls, entry: 'ServerVariableRegistryEntry'):
+        """
+        Internal method to get the sequence number of a server variable based in its registry entry.
+        """
+        key = cls.get_key(entry.backend.scope)
+        return await entry.backend.get_sequence_number(key)
+
+    @classmethod
+    async def get_tabular_data(
+        cls,
+        entry: 'ServerVariableRegistryEntry',
+        filters: Optional[FilterQuery] = None,
+        pagination: Optional[Pagination] = None,
+    ) -> DataResponse:
+        """
+        Internal method to get tabular data from the backend
+        """
+        key = cls.get_key(entry.backend.scope)
+        data, count = await entry.backend.read_filtered(key, filters, pagination)
+        if data is None:
+            return DataResponse(data=None, count=0, schema=None)
+        return build_data_response(data, count)
+
+    @classmethod
+    def get_key(cls, scope: Literal['global', 'user']):
+        """
+        Resolve the key for the given scope
+
+        :param scope: the scope to resolve the key for
+        """
+        if scope == 'global':
+            return 'global'
+
+        user = USER.get()
+
+        if user:
+            return user.identity_id
+
+        raise ValueError('User not found when trying to compute the key for a user-scoped store')
+
+    @property
+    def key(self):
+        """
+        Current key for the backend
+        """
+        return self.get_key(self.scope)
+
+    @classmethod
+    async def _notify(cls, uid: str, key: str, backend: ServerBackend):
+        """
+        Internal method to notify clients of a change in the value
+
+        :param uid: the uid of the variable
+        :param key: the key for the backend
+        :param backend: the backend instance
+        """
+        from dara.core.internal.registries import utils_registry
+
+        ws_mgr: WebsocketManager = utils_registry.get('WebsocketManager')
+
+        message = ServerVariableMessage(uid=uid, sequence_number=await backend.get_sequence_number(key))
+
+        if backend.scope == 'global':
+            return await ws_mgr.broadcast(message)
+
+        user = USER.get()
+        assert user is not None, 'User not found when trying to send notification'
+        user_id = user.identity_id
+        return await ws_mgr.send_message_to_user(user_id, message)
+
+    def update(self, value: Any):
+        """
+        Create an action to update the value of this Variable to a provided value.
+
+        ```python
+        import pandas as pd
+        from dara.core import ServerVariable
+        from dara.components import Button
+
+        data = ServerVariable(pd.DataFrame({'a': [1, 2, 3]}))
+
+        Button(
+            'Empty Data',
+            onclick=data.update(None),
+        )
+
+        ```
+        """
+        from dara.core.interactivity.actions import UpdateVariableImpl
+
+        return UpdateVariableImpl(variable=self, value=value)
+
+    def reset(self):
+        raise NotImplementedError('ServerVariable does not support reset')
+
+    async def read(self):
+        """
+        Read the current value from the backend.
+        Depending on the scope, the value will be global or user-specific.
+        """
+        return await self.backend.read(self.key)
+
+    async def write(self, value: Any):
+        """
+        Write a new value to the backend.
+        Depending on the scope, the value will be global or user-specific.
+
+        :param value: the new value to write
+        """
+        value = await self.backend.write(self.key, value)
+        await self._notify(self.uid, self.key, self.backend)
+        return value
+
+    async def read_filtered(
+        self, filters: Optional[Union[FilterQuery, dict]] = None, pagination: Optional[Pagination] = None
+    ):
+        """
+        Read a filtered value from the backend.
+        Depending on the scope, the value will be global or user-specific.
+
+        :param filters: the filters to apply
+        :param pagination: the pagination to apply
+        """
+        return await self.backend.read_filtered(self.key, filters, pagination)
+
+    @model_serializer(mode='wrap')
+    def ser_model(self, nxt: SerializerFunctionWrapHandler) -> dict:
+        parent_dict = nxt(self)
+        return {**parent_dict, '__typename': 'ServerVariable', 'uid': str(parent_dict['uid'])}
+
+
+class ServerVariableRegistryEntry(CachedRegistryEntry):
+    """
+    Registry entry for ServerVariable.
+    """
+
+    backend: ServerBackend
+    """
+    Backend instance
+    """
+
+    model_config = ConfigDict(extra='forbid', arbitrary_types_allowed=True)