dara-core 1.20.0__py3-none-any.whl → 1.20.1a1__py3-none-any.whl

dara/core/internal/routing.py

@@ -16,13 +16,14 @@ limitations under the License.
 """
 
 import inspect
+import json
 import os
-from collections.abc import Mapping
 from functools import wraps
 from importlib.metadata import version
-from typing import Any, Callable, Dict, List, Optional
+from typing import Any, Callable, Dict, List, Mapping, Optional
 
 import anyio
+import pandas
 from fastapi import (
     APIRouter,
     Body,
@@ -38,28 +39,25 @@ from fastapi.responses import StreamingResponse
 from pandas import DataFrame
 from pydantic import BaseModel
 from starlette.background import BackgroundTask
-from starlette.status import HTTP_415_UNSUPPORTED_MEDIA_TYPE
 
 from dara.core.auth.routes import verify_session
-from dara.core.base_definitions import ActionResolverDef, BaseTask, NonTabularDataError, UploadResolverDef
+from dara.core.base_definitions import ActionResolverDef, BaseTask, UploadResolverDef
 from dara.core.configuration import Configuration
-from dara.core.interactivity.any_data_variable import upload
+from dara.core.interactivity.any_data_variable import DataVariableRegistryEntry, upload
 from dara.core.interactivity.filtering import FilterQuery, Pagination
-from dara.core.interactivity.server_variable import ServerVariable
 from dara.core.internal.cache_store import CacheStore
 from dara.core.internal.download import DownloadRegistryEntry
 from dara.core.internal.execute_action import CURRENT_ACTION_ID
 from dara.core.internal.normalization import NormalizedPayload, denormalize, normalize
-from dara.core.internal.pandas_utils import data_response_to_json, df_to_json, is_data_response
+from dara.core.internal.pandas_utils import df_to_json
 from dara.core.internal.registries import (
     action_def_registry,
     action_registry,
     backend_store_registry,
     component_registry,
+    data_variable_registry,
     derived_variable_registry,
-    download_code_registry,
     latest_value_registry,
-    server_variable_registry,
     static_kwargs_registry,
     template_registry,
     upload_resolver_registry,
@@ -88,7 +86,7 @@ def error_decorator(handler: Callable[..., Any]):
                 if isinstance(err, HTTPException):
                     raise err
                 dev_logger.error('Unhandled error', error=err)
-                raise HTTPException(status_code=500, detail=str(err)) from err
+                raise HTTPException(status_code=500, detail=str(err))
 
         return _async_inner_func
 
@@ -101,7 +99,7 @@ def error_decorator(handler: Callable[..., Any]):
             if isinstance(err, HTTPException):
                 raise err
             dev_logger.error('Unhandled error', error=err)
-            raise HTTPException(status_code=500, detail=str(err)) from err
+            raise HTTPException(status_code=500, detail=str(err))
 
     return _inner_func
 
@@ -115,7 +113,7 @@ def create_router(config: Configuration):
     core_api_router = APIRouter()
 
     @core_api_router.get('/actions', dependencies=[Depends(verify_session)])
-    async def get_actions():
+    async def get_actions():   # pylint: disable=unused-variable
         return action_def_registry.get_all().items()
 
     class ActionRequestBody(BaseModel):
@@ -135,7 +133,7 @@ def create_router(config: Configuration):
         """Execution id, unique to this request"""
 
     @core_api_router.post('/action/{uid}', dependencies=[Depends(verify_session)])
-    async def get_action(uid: str, body: ActionRequestBody):
+    async def get_action(uid: str, body: ActionRequestBody):  # pylint: disable=unused-variable
         store: CacheStore = utils_registry.get('Store')
         task_mgr: TaskManager = utils_registry.get('TaskManager')
         registry_mgr: RegistryLookup = utils_registry.get('RegistryLookup')
@@ -152,14 +150,7 @@ def create_router(config: Configuration):
 
         # Execute the action - kick off a background task to run the handler
         response = await action_def.execute_action(
-            action_def,
-            body.input,
-            values,
-            static_kwargs,
-            body.execution_id,
-            body.ws_channel,
-            store,
-            task_mgr,
+            action_def, body.input, values, static_kwargs, body.execution_id, body.ws_channel, store, task_mgr
         )
 
         if isinstance(response, BaseTask):
@@ -168,22 +159,15 @@ def create_router(config: Configuration):
 
         return {'execution_id': response}
 
-    @core_api_router.get('/download')  # explicitly unauthenticated
-    async def get_download(code: str):
+    @core_api_router.get('/download')
+    async def get_download(code: str):   # pylint: disable=unused-variable
         store: CacheStore = utils_registry.get('Store')
 
         try:
             data_entry = await store.get(DownloadRegistryEntry, key=code)
 
-            # If not found directly in the store, use the override registry
-            # to check if we can get the download entry from there
             if data_entry is None:
-                registry_mgr: RegistryLookup = utils_registry.get('RegistryLookup')
-                # NOTE: This will throw a Value/KeyError if the code is not found so no need to rethrow
-                data_entry = await registry_mgr.get(download_code_registry, code)
-                # We managed to find one from the lookup,
-                # remove it from the registry immediately because it's one time use
-                download_code_registry.remove(code)
+                raise ValueError('Invalid or expired download code')
 
             async_file, cleanup = await data_entry.download(data_entry)
 
@@ -204,11 +188,11 @@ def create_router(config: Configuration):
                 background=BackgroundTask(cleanup),
             )
 
-        except (KeyError, ValueError) as e:
-            raise ValueError('Invalid or expired download code') from e
+        except KeyError:
+            raise ValueError('Invalid or expired download code')
 
     @core_api_router.get('/config', dependencies=[Depends(verify_session)])
-    async def get_config():
+    async def get_config():  # pylint: disable=unused-variable
         return {
             **config.model_dump(
                 include={
@@ -225,13 +209,13 @@ def create_router(config: Configuration):
         }
 
     @core_api_router.get('/auth-config')
-    async def get_auth_config():
+    async def get_auth_config():  # pylint: disable=unused-variable
         return {
             'auth_components': config.auth_config.component_config.model_dump(),
         }
 
     @core_api_router.get('/components', dependencies=[Depends(verify_session)])
-    async def get_components(name: Optional[str] = None):
+    async def get_components(name: Optional[str] = None):  # pylint: disable=unused-variable
         """
         If name is passed, will try to register the component
 
@@ -252,7 +236,7 @@ def create_router(config: Configuration):
         ws_channel: str
 
     @core_api_router.post('/components/{component}', dependencies=[Depends(verify_session)])
-    async def get_component(component: str, body: ComponentRequestBody):
+    async def get_component(component: str, body: ComponentRequestBody):  # pylint: disable=unused-variable
         CURRENT_COMPONENT_ID.set(body.uid)
         WS_CHANNEL.set(body.ws_channel)
         store: CacheStore = utils_registry.get('Store')
@@ -281,7 +265,7 @@ def create_router(config: Configuration):
         raise HTTPException(status_code=400, detail='Requesting this type of component is not supported')
 
     @core_api_router.get('/derived-variable/{uid}/latest', dependencies=[Depends(verify_session)])
-    async def get_latest_derived_variable(uid: str):
+    async def get_latest_derived_variable(uid: str):  # pylint: disable=unused-variable
         try:
             store: CacheStore = utils_registry.get('Store')
             latest_value_entry = latest_value_registry.get(uid)
@@ -305,67 +289,129 @@ def create_router(config: Configuration):
             return latest_value
 
         except KeyError as err:
-            raise ValueError(f'Could not find latest value for derived variable with uid: {uid}') from err
+            raise ValueError(f'Could not find latest value for derived variable with uid: {uid}').with_traceback(
+                err.__traceback__
+            )
 
-    class TabularRequestBody(BaseModel):
+    class DataVariableRequestBody(BaseModel):
         filters: Optional[FilterQuery] = None
-        ws_channel: str
-        dv_values: Optional[NormalizedPayload[List[Any]]] = None
-        """DerivedVariable values if variable is a DerivedVariable"""
-        force_key: Optional[str] = None
-        """Optional force key if variable is a DerivedVariable and a recalculation is forced"""
+        cache_key: Optional[str] = None
+        ws_channel: Optional[str] = None
 
-    @core_api_router.post('/tabular-variable/{uid}', dependencies=[Depends(verify_session)])
-    async def get_tabular_variable(
+    @core_api_router.post('/data-variable/{uid}', dependencies=[Depends(verify_session)])
+    async def get_data_variable(
         uid: str,
-        body: TabularRequestBody,
+        body: DataVariableRequestBody,
         offset: Optional[int] = None,
         limit: Optional[int] = None,
         order_by: Optional[str] = None,
         index: Optional[str] = None,
-    ):
-        """
-        Generic endpoint for getting tabular data from a variable.
-        Supports ServerVariables and DerivedVariables.
-        """
-        WS_CHANNEL.set(body.ws_channel)
-
+    ):   # pylint: disable=unused-variable
         try:
-            pagination = Pagination(offset=offset, limit=limit, orderBy=order_by, index=index)
+            store: CacheStore = utils_registry.get('Store')
+            task_mgr: TaskManager = utils_registry.get('TaskManager')
             registry_mgr: RegistryLookup = utils_registry.get('RegistryLookup')
+            data_variable_entry: DataVariableRegistryEntry = await registry_mgr.get(data_variable_registry, uid)
+
+            data = None
+            WS_CHANNEL.set(body.ws_channel)
+
+            if data_variable_entry.type == 'derived':
+                if body.cache_key is None:
+                    raise HTTPException(status_code=400, detail='Cache key is required for derived data variables')
+
+                if body.ws_channel is None:
+                    raise HTTPException(
+                        status_code=400, detail='Websocket channel is required for derived data variables'
+                    )
+
+                derived_variable_entry = await registry_mgr.get(derived_variable_registry, uid)
+
+                data = await data_variable_entry.get_data(
+                    derived_variable_entry,
+                    data_variable_entry,
+                    body.cache_key,
+                    store,
+                    body.filters,
+                    Pagination(offset=offset, limit=limit, orderBy=order_by, index=index),
+                    format_for_display=True,
+                )
+                if isinstance(data, BaseTask):
+                    await task_mgr.run_task(data, body.ws_channel)
+                    return {'task_id': data.task_id}
+            elif data_variable_entry.type == 'plain':
+                data = await data_variable_entry.get_data(
+                    data_variable_entry,
+                    store,
+                    body.filters,
+                    Pagination(offset=offset, limit=limit, orderBy=order_by, index=index),
+                    format_for_display=True,
+                )
 
-            # ServerVariable
-            if body.dv_values is None:
-                server_variable_entry = await registry_mgr.get(server_variable_registry, uid)
-                data_response = await ServerVariable.get_tabular_data(server_variable_entry, body.filters, pagination)
-                return Response(data_response_to_json(data_response), media_type='application/json')
+            dev_logger.debug(
+                f'DataVariable {data_variable_entry.uid[:3]}..{data_variable_entry.uid[-3:]}',
+                'return value',
+                {'value': data.describe() if isinstance(data, pandas.DataFrame) else None, 'uid': uid},  # type: ignore
+            )
+
+            if data is None:
+                return None
 
-            # DerivedVariable
+            # Explicitly convert to JSON to avoid implicit serialization;
+            # return as records as that makes more sense in a JSON structure
+            return Response(
+                content=df_to_json(data) if isinstance(data, pandas.DataFrame) else data, media_type='application/json'
+            )   # type: ignore
+        except ValueError as e:
+            raise HTTPException(status_code=400, detail=str(e))
+
+    class DataVariableCountRequestBody(BaseModel):
+        cache_key: Optional[str] = None
+        filters: Optional[FilterQuery] = None
+
+    @core_api_router.post('/data-variable/{uid}/count', dependencies=[Depends(verify_session)])
+    async def get_data_variable_count(uid: str, body: Optional[DataVariableCountRequestBody] = None):
+        try:
             store: CacheStore = utils_registry.get('Store')
-            task_mgr: TaskManager = utils_registry.get('TaskManager')
-            variable_def = await registry_mgr.get(derived_variable_registry, uid)
-            values = denormalize(body.dv_values.data, body.dv_values.lookup)
+            registry_mgr: RegistryLookup = utils_registry.get('RegistryLookup')
+            variable_def = await registry_mgr.get(data_variable_registry, uid)
 
-            result = await variable_def.get_tabular_data(
-                variable_def, store, task_mgr, values, body.force_key, pagination, body.filters
-            )
+            if variable_def.type == 'plain':
+                return await variable_def.get_total_count(
+                    variable_def, store, body.filters if body is not None else None
+                )
 
-            if isinstance(result, BaseTask):
-                await task_mgr.run_task(result, body.ws_channel)
-                return {'task_id': result.task_id}
+            if body is None or body.cache_key is None:
+                raise HTTPException(
+                    status_code=400, detail="Cache key is required when requesting DerivedDataVariable's count"
+                )
 
-            return Response(data_response_to_json(result), media_type='application/json')
-        except NonTabularDataError as e:
-            raise HTTPException(status_code=HTTP_415_UNSUPPORTED_MEDIA_TYPE, detail=str(e)) from e
+            return await variable_def.get_total_count(variable_def, store, body.cache_key, body.filters)
+        except ValueError as e:
+            raise HTTPException(status_code=400, detail=str(e))
 
-    @core_api_router.get('/server-variable/{uid}/sequence', dependencies=[Depends(verify_session)])
-    async def get_server_variable_sequence(
-        uid: str,
-    ):
-        registry_mgr: RegistryLookup = utils_registry.get('RegistryLookup')
-        server_variable_entry = await registry_mgr.get(server_variable_registry, uid)
-        seq_num = await ServerVariable.get_sequence_number(server_variable_entry)
-        return {'sequence_number': seq_num}
+    @core_api_router.get('/data-variable/{uid}/schema', dependencies=[Depends(verify_session)])
+    async def get_data_variable_schema(uid: str, cache_key: Optional[str] = None):
+        try:
+            store: CacheStore = utils_registry.get('Store')
+            registry_mgr: RegistryLookup = utils_registry.get('RegistryLookup')
+            data_def = await registry_mgr.get(data_variable_registry, uid)
+
+            if data_def.type == 'plain':
+                return await data_def.get_schema(data_def, store)
+
+            if cache_key is None:
+                raise HTTPException(
+                    status_code=400, detail='Cache key is required when requesting DerivedDataVariable schema'
+                )
+
+            # Use the other registry for derived variables
+            derived_ref = await registry_mgr.get(derived_variable_registry, uid)
+            data = await data_def.get_schema(derived_ref, store, cache_key)
+            content = json.dumps(jsonable_encoder(data)) if isinstance(data, dict) else data
+            return Response(content=content, media_type='application/json')
+        except ValueError as e:
+            raise HTTPException(status_code=400, detail=str(e))
 
     @core_api_router.post('/data/upload', dependencies=[Depends(verify_session)])
     async def upload_data(
@@ -381,10 +427,7 @@ def create_router(config: Configuration):
         registry_mgr: RegistryLookup = utils_registry.get('RegistryLookup')
 
         if data_uid is None and resolver_id is None:
-            raise HTTPException(
-                400,
-                'Neither resolver_id or data_uid specified, at least one of them is required',
-            )
+            raise HTTPException(400, 'Neither resolver_id or data_uid specified, at least one of them is required')
 
         try:
             # If resolver id is provided, run the custom
@@ -397,15 +440,16 @@ def create_router(config: Configuration):
 
             return {'status': 'SUCCESS'}
         except Exception as e:
-            raise HTTPException(status_code=400, detail=str(e)) from e
+            raise HTTPException(status_code=400, detail=str(e))
 
     class DerivedStateRequestBody(BaseModel):
         values: NormalizedPayload[List[Any]]
-        force_key: Optional[str] = None
+        force: bool
         ws_channel: str
+        is_data_variable: Optional[bool] = False
 
     @core_api_router.post('/derived-variable/{uid}', dependencies=[Depends(verify_session)])
-    async def get_derived_variable(uid: str, body: DerivedStateRequestBody):
+    async def get_derived_variable(uid: str, body: DerivedStateRequestBody):  # pylint: disable=unused-variable
         task_mgr: TaskManager = utils_registry.get('TaskManager')
         store: CacheStore = utils_registry.get('Store')
         registry_mgr: RegistryLookup = utils_registry.get('RegistryLookup')
@@ -413,7 +457,7 @@ def create_router(config: Configuration):
 
         values = denormalize(body.values.data, body.values.lookup)
 
-        result = await variable_def.get_value(variable_def, store, task_mgr, values, body.force_key)
+        result = await variable_def.get_value(variable_def, store, task_mgr, values, body.force)
 
         response: Any = result
 
@@ -422,10 +466,7 @@ def create_router(config: Configuration):
         if isinstance(result['value'], BaseTask):
             # Kick off the task
             await task_mgr.run_task(result['value'], body.ws_channel)
-            response = {
-                'task_id': result['value'].task_id,
-                'cache_key': result['cache_key'],
-            }
+            response = {'task_id': result['value'].task_id, 'cache_key': result['cache_key']}
 
         dev_logger.debug(
             f'DerivedVariable {variable_def.uid[:3]}..{variable_def.uid[-3:]}',
@@ -471,7 +512,7 @@ def create_router(config: Configuration):
                 tg.start_soon(_write, store_uid, value)
 
     @core_api_router.get('/tasks/{task_id}', dependencies=[Depends(verify_session)])
-    async def get_task_result(task_id: str):
+    async def get_task_result(task_id: str):   # pylint: disable=unused-variable
         try:
             task_mgr: TaskManager = utils_registry.get('TaskManager')
             res = await task_mgr.get_result(task_id)
@@ -482,35 +523,30 @@ def create_router(config: Configuration):
                 {'value': res},
             )
 
-            # Serialize dataframes correctly, either direct or as a DataResponse
+            # Serialize dataframes correctly
             if isinstance(res, DataFrame):
-                return Response(df_to_json(res), media_type='application/json')
-            elif is_data_response(res):
-                return Response(data_response_to_json(res), media_type='application/json')
+                return Response(df_to_json(res))
 
             return res
-        except Exception as err:
-            raise ValueError(f'The result for task id {task_id} could not be found') from err
+        except Exception as e:
+            raise ValueError(f'The result for task id {task_id} could not be found').with_traceback(e.__traceback__)
 
     @core_api_router.delete('/tasks/{task_id}', dependencies=[Depends(verify_session)])
-    async def cancel_task(task_id: str):
+    async def cancel_task(task_id: str):   # pylint: disable=unused-variable
         try:
             task_mgr: TaskManager = utils_registry.get('TaskManager')
             return await task_mgr.cancel_task(task_id)
         except TaskManagerError as e:
-            dev_logger.error(
-                f'The task id {task_id} could not be found, it may have already been cancelled',
-                e,
-            )
+            dev_logger.error(f'The task id {task_id} could not be found, it may have already been cancelled', e)
 
     @core_api_router.get('/template/{template}', dependencies=[Depends(verify_session)])
-    async def get_template(template: str):
+    async def get_template(template: str):  # pylint: disable=unused-variable
         try:
             selected_template = template_registry.get(template)
             normalized_template, lookup = normalize(jsonable_encoder(selected_template))
             return {'data': normalized_template, 'lookup': lookup}
-        except KeyError as err:
-            raise HTTPException(status_code=404, detail=f'Template: {template}, not found in registry') from err
+        except KeyError:
+            raise HTTPException(status_code=404, detail=f'Template: {template}, not found in registry')
         except Exception as e:
             dev_logger.error('Something went wrong while trying to get the template', e)
 

dara/core/internal/scheduler.py

@@ -20,7 +20,7 @@ from datetime import datetime
 from multiprocessing import get_context
 from multiprocessing.process import BaseProcess
 from pickle import PicklingError
-from typing import Any, List, Optional, Union, cast
+from typing import Any, List, Optional, Union
 
 from croniter import croniter
 from pydantic import BaseModel, field_validator
@@ -56,28 +56,25 @@ class ScheduledJob(BaseModel):
             job_process = ctx.Process(target=self._refresh_timer, args=(func, args), daemon=True)
             job_process.start()
             return job_process
-        except PicklingError as err:
+        except PicklingError:
             raise PicklingError(
                 """
             Unable to pickle scheduled function. Please ensure that the function you are trying
             to schedule is not in the same file as the ConfigurationBuilder is defined and that
             the function is not a lambda.
             """
-            ) from err
+            )
 
     def _refresh_timer(self, func, args):
         while self.continue_running and not (self.run_once and not self.first_execution):
-            interval: int
+            interval = self.interval
             # If there's more than one interval to wait, i.e. this is a weekday process
-            if isinstance(self.interval, list):
+            if type(interval) == list:
                 # Wait the first interval if this is the first execution of the job
                 interval = self.interval[0] if self.first_execution else self.interval[1]
-            else:
-                interval = self.interval
-
             self.first_execution = False
             # Wait the interval and then run the job
-            time.sleep(cast(int, interval))
+            time.sleep(interval)
             func(*args)
 
 
@@ -179,7 +176,7 @@ class ScheduledJobFactory(BaseModel):
 
     @field_validator('weekday', mode='before')
     @classmethod
-    def validate_weekday(cls, weekday: Any) -> datetime:
+    def validate_weekday(cls, weekday: Any) -> datetime:  # pylint: disable=E0213
         if isinstance(weekday, datetime):
             return weekday
         if isinstance(weekday, str):
@@ -286,9 +283,7 @@ class Scheduler:
     def _weekday(self, weekday: int):
         # The job must execute on a weekly interval
         return ScheduledJobFactory(
-            interval=self.interval * 604800,
-            run_once=self._run_once,
-            weekday=str(weekday),  # type: ignore
+            interval=self.interval * 604800, run_once=self._run_once, weekday=str(weekday)  # type: ignore
         )
 
     def monday(self) -> ScheduledJobFactory:

dara/core/internal/settings.py

@@ -74,7 +74,7 @@ def generate_env_file(filename='.env'):
         f.write(env_content)
 
 
-@lru_cache
+@lru_cache()
 def get_settings():
     """
     Get a cached instance of the settings, loading values from the .env if present.
@@ -83,7 +83,7 @@ def get_settings():
 
     # Test purposes - if DARA_TEST_FLAG is set then override env with .env.test
     if os.environ.get('DARA_TEST_FLAG', None) is not None:
-        return Settings(**dotenv_values('.env.test'))  # type: ignore
+        return Settings(**dotenv_values('.env.test'))
 
     env_error = False
 

dara/core/internal/store.py

@@ -17,7 +17,7 @@ limitations under the License.
 
 from typing import Any, Dict, List, Optional
 
-from dara.core.base_definitions import CacheType, PendingTask
+from dara.core.base_definitions import CacheType, PendingTask, PendingValue
 from dara.core.internal.utils import get_cache_scope
 
 
@@ -59,6 +59,8 @@ class Store:
         cache_key = get_cache_scope(cache_type)
         value = self._store.get(cache_key, {}).get(key)
 
+        if isinstance(value, PendingValue):
+            return await value.wait()
         if isinstance(value, PendingTask):
             return await value.run()
         return value
@@ -84,8 +86,33 @@ class Store:
         if self._store.get(cache_key) is None:
             self._store[cache_key] = {}
 
+        # If there is a PendingValue set for this key then trigger its resolution
+        if isinstance(self._store[cache_key].get(key), PendingValue):
+            if error is not None:
+                self._store[cache_key][key].error(error)
+            else:
+                self._store[cache_key][key].resolve(value)
+
         self._store[cache_key][key] = value
 
+    def set_pending_value(self, key: str, cache_type: Optional[CacheType] = CacheType.GLOBAL):
+        """
+        Set a pending state for a value in the store. This will trigger the async behavior of the get call if subsequent
+        requests ask for the same key. A future is created in the store, which all requests then listen for the
+        resolution of before returning.
+
+        :param key: the key to set as pending
+        :param cache_type: whether to pull the value from the specified cache specific store or the global one, defaults to
+                            the global one
+        """
+        cache_key = get_cache_scope(cache_type)
+        if self._store.get(cache_key) is None:
+            self._store[cache_key] = {}
+
+        pending_val = PendingValue()
+
+        self._store[cache_key][key] = pending_val
+
     def set_pending_task(self, key: str, pending_task: PendingTask, cache_type: Optional[CacheType] = CacheType.GLOBAL):
         """
         Store a pending task state for a given key in the store. This will trigger the async behavior of the get call if subsequent
@@ -131,7 +158,7 @@ class Store:
                 # Otherwise go through and remove any non-pending values
                 keys = list(cache_type_store.keys())
                 for key in keys:
-                    if not isinstance(cache_type_store[key], PendingTask):
+                    if not isinstance(cache_type_store[key], (PendingValue, PendingTask)):
                         cache_type_store.pop(key)
 
     def list(self, cache_type: Optional[CacheType] = CacheType.GLOBAL) -> List[str]: