dara-core 1.20.0a1__py3-none-any.whl → 1.20.1a1__py3-none-any.whl

dara/core/internal/normalization.py

@@ -15,11 +15,11 @@ See the License for the specific language governing permissions and
 limitations under the License.
 """
 
-from collections.abc import Mapping
 from typing import (
     Any,
     Generic,
     List,
+    Mapping,
     Optional,
     Tuple,
     TypeVar,
@@ -31,6 +31,7 @@ from typing import (
 from typing_extensions import TypedDict, TypeGuard
 
 from dara.core.base_definitions import DaraBaseModel as BaseModel
+from dara.core.internal.hashing import hash_object
 
 JsonLike = Union[Mapping, List]
 
@@ -47,7 +48,7 @@ class Placeholder(TypedDict):
     Placeholder object 'Referrable' objects are replaced with
     """
 
-    __ref: str
+    __ref: str   # pylint: disable=unused-private-member
 
 
 class Referrable(TypedDict):
@@ -55,7 +56,7 @@ class Referrable(TypedDict):
     Describes an object which can be replaced by a Placeholder.
     """
 
-    __typename: str
+    __typename: str   # pylint: disable=unused-private-member
     uid: str
 
 
@@ -80,6 +81,10 @@ def _get_identifier(obj: Referrable) -> str:
         nested = ','.join(cast(List[str], obj['nested']))
         identifier = f'{identifier}:{nested}'
 
+    if _is_referrable_with_filters(obj):
+        filter_hash = hash_object(obj['filters'])
+        identifier = f'{identifier}:{filter_hash}'
+
     return identifier
 
 
@@ -128,11 +133,13 @@ def _loop(iterable: JsonLike):
 
 
 @overload
-def normalize(obj: Mapping, check_root: bool = True) -> Tuple[Mapping, Mapping]: ...
+def normalize(obj: Mapping, check_root: bool = True) -> Tuple[Mapping, Mapping]:
+    ...
 
 
 @overload
-def normalize(obj: List, check_root: bool = True) -> Tuple[List, Mapping]: ...
+def normalize(obj: List, check_root: bool = True) -> Tuple[List, Mapping]:
+    ...
 
 
 def normalize(obj: JsonLike, check_root: bool = True) -> Tuple[JsonLike, Mapping]:
@@ -162,7 +169,7 @@ def normalize(obj: JsonLike, check_root: bool = True) -> Tuple[JsonLike, Mapping
         for key, value in _loop(obj):
             # For iterables, recursively call normalize
             if isinstance(value, (dict, list)):
-                _normalized, _lookup = normalize(value)  # type: ignore
+                _normalized, _lookup = normalize(value)   # type: ignore
                 output[key] = _normalized  # type: ignore
                 lookup.update(_lookup)
             else:
@@ -173,11 +180,13 @@ def normalize(obj: JsonLike, check_root: bool = True) -> Tuple[JsonLike, Mapping
 
 
 @overload
-def denormalize(normalized_obj: Mapping, lookup: Mapping) -> Mapping: ...
+def denormalize(normalized_obj: Mapping, lookup: Mapping) -> Mapping:
+    ...
 
 
 @overload
-def denormalize(normalized_obj: List, lookup: Mapping) -> List: ...
+def denormalize(normalized_obj: List, lookup: Mapping) -> List:
+    ...
 
 
 def denormalize(normalized_obj: JsonLike, lookup: Mapping) -> Optional[JsonLike]:
@@ -197,7 +206,7 @@ def denormalize(normalized_obj: JsonLike, lookup: Mapping) -> Optional[JsonLike]
     # Whole object is a placeholder
     if _is_placeholder(normalized_obj):
         ref = normalized_obj['__ref']
-        referrable = lookup.get(ref, None)
+        referrable = lookup[ref] if ref in lookup else None
 
         if isinstance(referrable, (list, dict)):
             return denormalize(referrable, lookup)

dara/core/internal/pandas_utils.py

@@ -15,24 +15,13 @@ See the License for the specific language governing permissions and
 limitations under the License.
 """
 
-import json
-import uuid
-from typing import Any, Literal, Optional, TypeVar, Union, cast, overload
+from typing import Optional, TypeVar
 
-from pandas import DataFrame, MultiIndex, Series
-from typing_extensions import TypedDict, TypeGuard
+from pandas import DataFrame, MultiIndex
 
 INDEX = '__index__'
 
 
-@overload
-def append_index(df: DataFrame) -> DataFrame: ...
-
-
-@overload
-def append_index(df: None) -> None: ...
-
-
 def append_index(df: Optional[DataFrame]) -> Optional[DataFrame]:
     """
     Add a numerical index column to the dataframe
@@ -42,7 +31,7 @@ def append_index(df: Optional[DataFrame]) -> Optional[DataFrame]:
 
     if INDEX not in df.columns:
         new_df = df.copy()
-        new_df.insert(0, INDEX, range(0, len(df.index)))  # type: ignore
+        new_df.insert(0, INDEX, range(0, len(df.index)))
         return new_df
 
     return df
@@ -58,7 +47,7 @@ def remove_index(value: value_type) -> value_type:
     Otherwise return same value untouched.
     """
     if isinstance(value, DataFrame):
-        return cast(value_type, value.drop(columns=['__index__'], inplace=False, errors='ignore'))
+        return value.drop(columns=['__index__'], inplace=False, errors='ignore')
 
     return value
 
@@ -76,12 +65,6 @@ def df_convert_to_internal(original_df: DataFrame) -> DataFrame:
     if any(isinstance(c, str) and c.startswith('__col__') for c in df.columns):
         return df
 
-    # Apply display transformations to the DataFrame
-    format_for_display(df)
-
-    # Append index to match the way we process the original DataFrame
-    df = cast(DataFrame, append_index(df))
-
     # Handle hierarchical columns: [(A, B), (A, C)] -> ['A_B', 'A_C']
     if isinstance(df.columns, MultiIndex):
         df.columns = ['_'.join(col).strip() if col[0] != INDEX else INDEX for col in df.columns.values]
@@ -107,89 +90,4 @@ def df_convert_to_internal(original_df: DataFrame) -> DataFrame:
 
 
 def df_to_json(df: DataFrame) -> str:
-    return df_convert_to_internal(df).to_json(orient='records', date_unit='ns') or ''
-
-
-def format_for_display(df: DataFrame) -> None:
-    """
-    Apply transformations to a DataFrame to make it suitable for display.
-    Not: this does NOT make a copy of the DataFrame
-    """
-    for col in df.columns:
-        column_data = df[col]
-        if isinstance(column_data, DataFrame):
-            # Handle duplicate column names - format each column in the sub-DataFrame
-            for sub_col in column_data.columns:
-                if isinstance(column_data[sub_col], Series) and column_data[sub_col].dtype == 'object':
-                    column_data.loc[:, sub_col] = column_data[sub_col].apply(str)
-        elif column_data.dtype == 'object':
-            # We need to convert all values to string to avoid issues with
-            # displaying data in the Table component, for example when
-            # displaying datetime and number objects in the same column
-            df.loc[:, col] = column_data.apply(str)
-
-
-class FieldType(TypedDict):
-    name: Union[str, tuple[str, ...]]
-    type: Literal['integer', 'number', 'boolean', 'datetime', 'duration', 'any', 'str']
-
-
-class DataFrameSchema(TypedDict):
-    fields: list[FieldType]
-    primaryKey: list[str]
-
-
-class DataResponse(TypedDict):
-    data: Optional[DataFrame]
-    count: int
-    schema: Optional[DataFrameSchema]
-
-
-def is_data_response(response: Any) -> TypeGuard[DataResponse]:
-    has_shape = isinstance(response, dict) and 'data' in response and 'count' in response
-    if not has_shape:
-        return False
-    return response['data'] is None or isinstance(response['data'], DataFrame)
-
-
-def data_response_to_json(response: DataResponse) -> str:
-    """
-    Serialize a DataResponse to JSON.
-
-    json.dumps() custom serializers only accept value->value mappings, whereas `to_json` on pandas returns a string directly.
-    To avoid double serialization, we first insert a placeholder string and then replace it with the actual serialized JSON.
-    """
-    placeholder = str(uuid.uuid4())
-
-    def _custom_serializer(obj: Any) -> Any:
-        if isinstance(obj, DataFrame):
-            return placeholder
-        raise TypeError(f'Object of type {type(obj)} is not JSON serializable')
-
-    result = json.dumps(response, default=_custom_serializer)
-    result = result.replace(
-        f'"{placeholder}"', df_to_json(response['data']) if response['data'] is not None else 'null'
-    )
-    return result
-
-
-def build_data_response(data: DataFrame, count: int) -> DataResponse:
-    data_internal = df_convert_to_internal(data)
-    schema = get_schema(data_internal)
-
-    return DataResponse(data=data, count=count, schema=schema)
-
-
-def get_schema(df: DataFrame):
-    from pandas.io.json._table_schema import build_table_schema
-
-    raw_schema = build_table_schema(df)
-
-    for field_data in cast(list, raw_schema['fields']):
-        if field_data.get('type') == 'datetime':
-            # for datetime fields we need to know the resolution, so we get the actual e.g. `datetime64[ns]` string
-            column_name = field_data.get('name')
-            dtype_str = str(df[column_name].dtype)
-            field_data['type'] = dtype_str
-
-    return cast(DataFrameSchema, raw_schema)
+    return df_convert_to_internal(df).to_json(orient='records') or ''

dara/core/internal/pool/definitions.py

@@ -95,7 +95,7 @@ class TaskDefinition:
     def __await__(self):
         """Await the underlying event, then return or raise the result"""
         yield from self.event.wait().__await__()
-        if isinstance(self.result, BaseException):
+        if isinstance(self.result, Exception):
             raise self.result
         return self.result
 

dara/core/internal/pool/task_pool.py

@@ -16,11 +16,10 @@ limitations under the License.
 """
 
 import atexit
-from collections.abc import Coroutine
 from contextlib import contextmanager
 from datetime import datetime
 from multiprocessing import active_children
-from typing import Any, Callable, Dict, Optional, Union, cast
+from typing import Any, Callable, Coroutine, Dict, Optional, cast
 
 import anyio
 from anyio.abc import TaskGroup
@@ -103,18 +102,16 @@ class TaskPool:
             try:
                 await wait_while(
                     lambda: self.status != PoolStatus.RUNNING
-                    or len(self.workers) != self.desired_workers
+                    or not len(self.workers) == self.desired_workers
                     or not all(w.status == WorkerStatus.IDLE for w in self.workers.values()),
                     timeout=timeout,
                 )
-            except TimeoutError as e:
-                raise RuntimeError('Failed to start pool') from e
+            except TimeoutError:
+                raise RuntimeError('Failed to start pool')
         else:
             raise RuntimeError('Pool already started')
 
-    def submit(
-        self, task_uid: str, function_name: str, args: Union[tuple, None] = None, kwargs: Union[dict, None] = None
-    ) -> TaskDefinition:
+    def submit(self, task_uid: str, function_name: str, args: tuple = (), kwargs: dict = {}) -> TaskDefinition:
         """
         Submit a new task to the pool
 
@@ -123,10 +120,6 @@ class TaskPool:
         :param args: list of arguments to pass to the function
         :param kwargs: dict of kwargs to pass to the function
         """
-        if args is None:
-            args = ()
-        if kwargs is None:
-            kwargs = {}
         self._check_pool_state()
 
         # Create a task definition to keep track of its progress
@@ -158,7 +151,7 @@ class TaskPool:
 
         task = self.tasks.pop(task_uid)
         if not task.event.is_set():
-            task.result = anyio.get_cancelled_exc_class()()
+            task.result = Exception('Task cancelled')
             task.event.set()
 
         # Task in progress, stop the worker
@@ -470,8 +463,9 @@ class TaskPool:
                 )
         elif is_log(worker_msg):
             dev_logger.info(f'Task: {worker_msg["task_uid"]}', {'logs': worker_msg['log']})
-        elif is_progress(worker_msg) and worker_msg['task_uid'] in self._progress_subscribers:
-            await self._progress_subscribers[worker_msg['task_uid']](worker_msg['progress'], worker_msg['message'])
+        elif is_progress(worker_msg):
+            if worker_msg['task_uid'] in self._progress_subscribers:
+                await self._progress_subscribers[worker_msg['task_uid']](worker_msg['progress'], worker_msg['message'])
 
     async def _wait_queue_depletion(self, timeout: Optional[float] = None):
         """
@@ -484,8 +478,8 @@ class TaskPool:
                 condition=lambda: self.status in (PoolStatus.CLOSED, PoolStatus.RUNNING) and len(self.tasks) > 0,
                 timeout=timeout,
             )
-        except TimeoutError as e:
-            raise TimeoutError('Tasks are still being executed') from e
+        except TimeoutError:
+            raise TimeoutError('Tasks are still being executed')
 
     async def _core_loop(self):
         """
@@ -499,14 +493,11 @@ class TaskPool:
             while self.status not in (PoolStatus.ERROR, PoolStatus.STOPPED):
                 await anyio.sleep(0.1)
 
-                try:
-                    self._handle_excess_workers()
-                    self._handle_orphaned_workers()
-                    self._handle_dead_workers()
-                    self._create_workers()
-                    await self._process_next_worker_message()
-                except Exception as e:
-                    dev_logger.error('Error in task pool', e)
+                self._handle_excess_workers()
+                self._handle_orphaned_workers()
+                self._handle_dead_workers()
+                self._create_workers()
+                await self._process_next_worker_message()
         finally:
             self.loop_stopped.set()
 

dara/core/internal/pool/utils.py

@@ -27,8 +27,6 @@ from typing import Any, Callable, Optional, Tuple
 import anyio
 from tblib import Traceback
 
-from dara.core.logging import dev_logger
-
 
 class SubprocessException:
     """
@@ -91,7 +89,9 @@ def read_from_shared_memory(pointer: SharedMemoryPointer) -> Any:
         data = shared_mem.buf[:data_size]
 
         # Unpickle and deepcopy
-        decoded_payload_shared = pickle.loads(shared_mem.buf)  # nosec B301 # we trust the shared memory pointer passed by the pool
+        decoded_payload_shared = pickle.loads(
+            shared_mem.buf
+        )   # nosec B301 # we trust the shared memory pointer passed by the pool
         decoded_payload = copy.deepcopy(decoded_payload_shared)
 
         # Cleanup
@@ -133,23 +133,20 @@ async def stop_process_async(process: BaseProcess, timeout: float = 3):
     # Terminate and wait for it to shutdown
     process.terminate()
 
-    try:
-        # mimic process.join() in an async way to not block
-        await wait_while(process.is_alive, timeout)
-
-        # If it's still alive
-        if process.is_alive():
-            try:
-                os.kill(process.pid, signal.SIGKILL)
-                await wait_while(process.is_alive, timeout)
-            except OSError as e:
-                raise RuntimeError(f'Unable to terminate subprocess with PID {process.pid}') from e
-
-        # If it's still alive raise an exception
-        if process.is_alive():
+    # mimic process.join() in an async way to not block
+    await wait_while(process.is_alive, timeout)
+
+    # If it's still alive
+    if process.is_alive():
+        try:
+            os.kill(process.pid, signal.SIGKILL)
+            await wait_while(process.is_alive, timeout)
+        except OSError:
             raise RuntimeError(f'Unable to terminate subprocess with PID {process.pid}')
-    except Exception as e:
-        dev_logger.error('Error stopping process', e)
+
+    # If it's still alive raise an exception
+    if process.is_alive():
+        raise RuntimeError(f'Unable to terminate subprocess with PID {process.pid}')
 
 
 def stop_process(process: BaseProcess, timeout: float = 3):
@@ -174,8 +171,8 @@ def stop_process(process: BaseProcess, timeout: float = 3):
         try:
             os.kill(process.pid, signal.SIGKILL)
             process.join(timeout)
-        except OSError as e:
-            raise RuntimeError(f'Unable to terminate subprocess with PID {process.pid}') from e
+        except OSError:
+            raise RuntimeError(f'Unable to terminate subprocess with PID {process.pid}')
 
     # If it's still alive raise an exception
     if process.is_alive():

dara/core/internal/pool/worker.py

@@ -57,8 +57,7 @@ class StdoutLogger:
             self.channel.worker_api.log(self.task_uid, msg)
 
     def flush(self):
-        if sys.__stdout__:
-            sys.__stdout__.flush()
+        sys.__stdout__.flush()
 
 
 def execute_function(func: Callable, args: tuple, kwargs: dict):
@@ -165,7 +164,7 @@ def worker_loop(worker_params: WorkerParameters, channel: Channel):
 
         # Redirect logs via the channel
         stdout_logger = StdoutLogger(task_uid, channel)
-        sys.stdout = stdout_logger  # type: ignore
+        sys.stdout = stdout_logger   # type: ignore
 
         try:
             payload_pointer = task['payload']

dara/core/internal/port_utils.py

@@ -27,7 +27,7 @@ def is_available(host: str, port: int) -> bool:
     """
     try:
         with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
-            sock.settimeout(2.0)  # timeout in case port is blocked
+            sock.settimeout(2.0)   # timeout in case port is blocked
             return sock.connect_ex((host, port)) != 0
     except BaseException:
         return False

dara/core/internal/registries.py

@@ -15,9 +15,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 """
 
-from collections.abc import Mapping
 from datetime import datetime
-from typing import Any, Callable, Set
+from typing import Any, Callable, Mapping, Set
 
 from dara.core.auth import BaseAuthConfig
 from dara.core.base_definitions import ActionDef, ActionResolverDef, UploadResolverDef
@@ -27,25 +26,23 @@ from dara.core.definitions import (
     EndpointConfiguration,
     Template,
 )
+from dara.core.interactivity.data_variable import DataVariableRegistryEntry
 from dara.core.interactivity.derived_variable import (
     DerivedVariableRegistryEntry,
     LatestValueRegistryEntry,
 )
-from dara.core.interactivity.server_variable import ServerVariableRegistryEntry
-from dara.core.internal.download import DownloadDataEntry
 from dara.core.internal.registry import Registry, RegistryType
 from dara.core.internal.websocket import CustomClientMessagePayload
 from dara.core.persistence import BackendStoreEntry
 
-action_def_registry = Registry[ActionDef](RegistryType.ACTION_DEF, CORE_ACTIONS)  # all registered actions
-action_registry = Registry[ActionResolverDef](RegistryType.ACTION)  # functions for actions requiring backend calls
+action_def_registry = Registry[ActionDef](RegistryType.ACTION_DEF, CORE_ACTIONS)   # all registered actions
+action_registry = Registry[ActionResolverDef](RegistryType.ACTION)   # functions for actions requiring backend calls
 upload_resolver_registry = Registry[UploadResolverDef](
     RegistryType.UPLOAD_RESOLVER
-)  # functions for upload resolvers requiring backend calls
+)   # functions for upload resolvers requiring backend calls
 component_registry = Registry[ComponentTypeAnnotation](RegistryType.COMPONENTS, CORE_COMPONENTS)
 config_registry = Registry[EndpointConfiguration](RegistryType.ENDPOINT_CONFIG)
-server_variable_registry = Registry[ServerVariableRegistryEntry](RegistryType.SERVER_VARIABLE, allow_duplicates=False)
-"""map of server variable uid -> server variable entry"""
+data_variable_registry = Registry[DataVariableRegistryEntry](RegistryType.DATA_VARIABLE, allow_duplicates=False)
 derived_variable_registry = Registry[DerivedVariableRegistryEntry](
     RegistryType.DERIVED_VARIABLE, allow_duplicates=False
 )
@@ -72,6 +69,3 @@ custom_ws_handlers_registry = Registry[Callable[[str, CustomClientMessagePayload
 
 backend_store_registry = Registry[BackendStoreEntry](RegistryType.BACKEND_STORE, allow_duplicates=False)
 """map of store uid -> store instance"""
-
-download_code_registry = Registry[DownloadDataEntry](RegistryType.DOWNLOAD_CODE, allow_duplicates=False)
-"""map of download codes -> download data entry, used only to allow overriding download code behaviour via RegistryLookup"""

dara/core/internal/registry.py

@@ -16,9 +16,8 @@ limitations under the License.
 """
 
 import copy
-from collections.abc import MutableMapping
 from enum import Enum
-from typing import Generic, Optional, TypeVar
+from typing import Generic, MutableMapping, Optional, TypeVar
 
 from dara.core.metrics import CACHE_METRICS_TRACKER, total_size
 
@@ -32,7 +31,7 @@ class RegistryType(str, Enum):
     DOWNLOAD = 'Download'
     COMPONENTS = 'Components'
     ENDPOINT_CONFIG = 'Endpoint Configuration'
-    SERVER_VARIABLE = 'ServerVariable'
+    DATA_VARIABLE = 'DataVariable'
     DERIVED_VARIABLE = 'DerivedVariable'
     LAST_VALUE = 'LatestValue'
     TEMPLATE = 'Template'
@@ -44,7 +43,6 @@ class RegistryType(str, Enum):
     PENDING_TOKENS = 'Pending tokens'
     CUSTOM_WS_HANDLERS = 'Custom WS handlers'
     BACKEND_STORE = 'Backend Store'
-    DOWNLOAD_CODE = 'Download Code'
 
 
 class Registry(Generic[T]):

dara/core/internal/registry_lookup.py

@@ -15,8 +15,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 """
 
-from collections.abc import Coroutine
-from typing import Callable, Dict, Literal, TypeVar, Union
+from typing import Callable, Coroutine, Dict, Literal
 
 from dara.core.internal.registry import Registry, RegistryType
 from dara.core.internal.utils import async_dedupe
@@ -24,30 +23,25 @@ from dara.core.internal.utils import async_dedupe
 RegistryLookupKey = Literal[
     RegistryType.ACTION,
     RegistryType.COMPONENTS,
+    RegistryType.DATA_VARIABLE,
     RegistryType.DERIVED_VARIABLE,
-    RegistryType.SERVER_VARIABLE,
     RegistryType.STATIC_KWARGS,
     RegistryType.UPLOAD_RESOLVER,
     RegistryType.BACKEND_STORE,
-    RegistryType.DOWNLOAD_CODE,
 ]
 CustomRegistryLookup = Dict[RegistryLookupKey, Callable[[str], Coroutine]]
 
-RegistryType = TypeVar('RegistryType')
-
 
 class RegistryLookup:
     """
     Manages registry Lookup.
     """
 
-    def __init__(self, handlers: Union[CustomRegistryLookup, None] = None):
-        if handlers is None:
-            handlers = {}
+    def __init__(self, handlers: CustomRegistryLookup = {}):
         self.handlers = handlers
 
     @async_dedupe
-    async def get(self, registry: Registry[RegistryType], uid: str) -> RegistryType:
+    async def get(self, registry: Registry, uid: str):
         """
         Get the entry from registry by uid.
         If uid is not in registry and it has a external handler that defined, will execute the handler
@@ -68,4 +62,4 @@ class RegistryLookup:
                 return entry
             raise ValueError(
                 f'Could not find uid {uid} in {registry.name} registry, did you register it before the app was initialized?'
-            ) from e
+            ).with_traceback(e.__traceback__)