dara-core 1.20.0__py3-none-any.whl → 1.20.1a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dara/core/__init__.py +0 -3
- dara/core/actions.py +2 -1
- dara/core/auth/basic.py +16 -22
- dara/core/auth/definitions.py +2 -2
- dara/core/auth/routes.py +5 -5
- dara/core/auth/utils.py +5 -5
- dara/core/base_definitions.py +64 -22
- dara/core/cli.py +7 -8
- dara/core/configuration.py +2 -5
- dara/core/css.py +2 -1
- dara/core/data_utils.py +19 -18
- dara/core/defaults.py +7 -6
- dara/core/definitions.py +19 -50
- dara/core/http.py +3 -7
- dara/core/interactivity/__init__.py +0 -6
- dara/core/interactivity/actions.py +50 -52
- dara/core/interactivity/any_data_variable.py +134 -7
- dara/core/interactivity/any_variable.py +8 -5
- dara/core/interactivity/data_variable.py +266 -8
- dara/core/interactivity/derived_data_variable.py +290 -7
- dara/core/interactivity/derived_variable.py +176 -416
- dara/core/interactivity/filtering.py +27 -46
- dara/core/interactivity/loop_variable.py +2 -2
- dara/core/interactivity/non_data_variable.py +68 -5
- dara/core/interactivity/plain_variable.py +15 -89
- dara/core/interactivity/switch_variable.py +19 -19
- dara/core/interactivity/url_variable.py +90 -10
- dara/core/internal/cache_store/base_impl.py +1 -2
- dara/core/internal/cache_store/cache_store.py +25 -22
- dara/core/internal/cache_store/keep_all.py +1 -4
- dara/core/internal/cache_store/lru.py +1 -5
- dara/core/internal/cache_store/ttl.py +1 -4
- dara/core/internal/cgroup.py +1 -1
- dara/core/internal/dependency_resolution.py +66 -60
- dara/core/internal/devtools.py +5 -12
- dara/core/internal/download.py +4 -13
- dara/core/internal/encoder_registry.py +7 -7
- dara/core/internal/execute_action.py +13 -13
- dara/core/internal/hashing.py +3 -1
- dara/core/internal/import_discovery.py +4 -3
- dara/core/internal/normalization.py +18 -9
- dara/core/internal/pandas_utils.py +5 -107
- dara/core/internal/pool/definitions.py +1 -1
- dara/core/internal/pool/task_pool.py +16 -25
- dara/core/internal/pool/utils.py +18 -21
- dara/core/internal/pool/worker.py +2 -3
- dara/core/internal/port_utils.py +1 -1
- dara/core/internal/registries.py +6 -12
- dara/core/internal/registry.py +2 -4
- dara/core/internal/registry_lookup.py +5 -11
- dara/core/internal/routing.py +145 -109
- dara/core/internal/scheduler.py +8 -13
- dara/core/internal/settings.py +2 -2
- dara/core/internal/store.py +29 -2
- dara/core/internal/tasks.py +195 -379
- dara/core/internal/utils.py +13 -36
- dara/core/internal/websocket.py +20 -21
- dara/core/js_tooling/js_utils.py +26 -28
- dara/core/js_tooling/templates/vite.config.template.ts +3 -12
- dara/core/logging.py +12 -13
- dara/core/main.py +11 -14
- dara/core/metrics/cache.py +1 -1
- dara/core/metrics/utils.py +3 -3
- dara/core/persistence.py +5 -27
- dara/core/umd/dara.core.umd.js +55428 -59098
- dara/core/visual/components/__init__.py +2 -2
- dara/core/visual/components/fallback.py +4 -30
- dara/core/visual/components/for_cmp.py +1 -4
- dara/core/visual/css/__init__.py +31 -30
- dara/core/visual/dynamic_component.py +28 -31
- dara/core/visual/progress_updater.py +3 -4
- {dara_core-1.20.0.dist-info → dara_core-1.20.1a1.dist-info}/METADATA +11 -12
- dara_core-1.20.1a1.dist-info/RECORD +114 -0
- dara/core/interactivity/client_variable.py +0 -71
- dara/core/interactivity/server_variable.py +0 -325
- dara/core/interactivity/state_variable.py +0 -69
- dara/core/interactivity/tabular_variable.py +0 -94
- dara/core/internal/multi_resource_lock.py +0 -70
- dara_core-1.20.0.dist-info/RECORD +0 -119
- {dara_core-1.20.0.dist-info → dara_core-1.20.1a1.dist-info}/LICENSE +0 -0
- {dara_core-1.20.0.dist-info → dara_core-1.20.1a1.dist-info}/WHEEL +0 -0
- {dara_core-1.20.0.dist-info → dara_core-1.20.1a1.dist-info}/entry_points.txt +0 -0
|
@@ -6,6 +6,7 @@ from dara.core.base_definitions import (
|
|
|
6
6
|
LruCachePolicy,
|
|
7
7
|
MostRecentCachePolicy,
|
|
8
8
|
PendingTask,
|
|
9
|
+
PendingValue,
|
|
9
10
|
TTLCachePolicy,
|
|
10
11
|
)
|
|
11
12
|
from dara.core.internal.cache_store.base_impl import CacheStoreImpl, PolicyT
|
|
@@ -22,7 +23,7 @@ def cache_impl_for_policy(policy: PolicyT) -> CacheStoreImpl[PolicyT]:
|
|
|
22
23
|
"""
|
|
23
24
|
impl: Optional[CacheStoreImpl] = None
|
|
24
25
|
|
|
25
|
-
if isinstance(policy, (
|
|
26
|
+
if isinstance(policy, LruCachePolicy) or isinstance(policy, MostRecentCachePolicy):
|
|
26
27
|
impl = LRUCache(policy)
|
|
27
28
|
elif isinstance(policy, TTLCachePolicy):
|
|
28
29
|
impl = TTLCache(policy)
|
|
@@ -61,24 +62,21 @@ class CacheScopeStore(Generic[PolicyT]):
|
|
|
61
62
|
|
|
62
63
|
return await cache.delete(key)
|
|
63
64
|
|
|
64
|
-
async def get(self, key: str, unpin: bool = False
|
|
65
|
+
async def get(self, key: str, unpin: bool = False) -> Optional[Any]:
|
|
65
66
|
"""
|
|
66
67
|
Retrieve an entry from the cache.
|
|
67
68
|
|
|
68
69
|
:param key: The key of the entry to retrieve.
|
|
69
70
|
:param unpin: If true, the entry will be unpinned if it is pinned.
|
|
70
|
-
:param raise_for_missing: If true, an exception will be raised if the entry is not found
|
|
71
71
|
"""
|
|
72
72
|
scope = get_cache_scope(self.policy.cache_type)
|
|
73
73
|
cache = self.caches.get(scope)
|
|
74
74
|
|
|
75
75
|
# No cache for this scope yet
|
|
76
76
|
if cache is None:
|
|
77
|
-
if raise_for_missing:
|
|
78
|
-
raise KeyError(f'No cache found for {scope}')
|
|
79
77
|
return None
|
|
80
78
|
|
|
81
|
-
return await cache.get(key, unpin=unpin
|
|
79
|
+
return await cache.get(key, unpin=unpin)
|
|
82
80
|
|
|
83
81
|
async def set(self, key: str, value: Any, pin: bool = False):
|
|
84
82
|
"""
|
|
@@ -152,30 +150,21 @@ class CacheStore:
|
|
|
152
150
|
|
|
153
151
|
return prev_entry
|
|
154
152
|
|
|
155
|
-
async def get(
|
|
156
|
-
self,
|
|
157
|
-
registry_entry: CachedRegistryEntry,
|
|
158
|
-
key: str,
|
|
159
|
-
unpin: bool = False,
|
|
160
|
-
raise_for_missing: bool = False,
|
|
161
|
-
) -> Optional[Any]:
|
|
153
|
+
async def get(self, registry_entry: CachedRegistryEntry, key: str, unpin: bool = False) -> Optional[Any]:
|
|
162
154
|
"""
|
|
163
155
|
Retrieve an entry from the cache for the given registry entry and cache key.
|
|
164
156
|
|
|
165
157
|
:param registry_entry: The registry entry to retrieve the value for.
|
|
166
158
|
:param key: The key of the entry to retrieve.
|
|
167
159
|
:param unpin: If true, the entry will be unpinned if it is pinned.
|
|
168
|
-
:param raise_for_missing: If true, an exception will be raised if the entry is not found
|
|
169
160
|
"""
|
|
170
161
|
registry_store = self.registry_stores.get(registry_entry.to_store_key())
|
|
171
162
|
|
|
172
163
|
# No store for this entry yet
|
|
173
164
|
if registry_store is None:
|
|
174
|
-
if raise_for_missing:
|
|
175
|
-
raise KeyError(f'No cache store found for {registry_entry.to_store_key()}')
|
|
176
165
|
return None
|
|
177
166
|
|
|
178
|
-
return await registry_store.get(key, unpin=unpin
|
|
167
|
+
return await registry_store.get(key, unpin=unpin)
|
|
179
168
|
|
|
180
169
|
async def get_or_wait(self, registry_entry: CachedRegistryEntry, key: str):
|
|
181
170
|
"""
|
|
@@ -188,6 +177,9 @@ class CacheStore:
|
|
|
188
177
|
|
|
189
178
|
value = await self.get(registry_entry, key)
|
|
190
179
|
|
|
180
|
+
if isinstance(value, PendingValue):
|
|
181
|
+
return await value.wait()
|
|
182
|
+
|
|
191
183
|
if isinstance(value, PendingTask):
|
|
192
184
|
return await value.run()
|
|
193
185
|
|
|
@@ -198,6 +190,7 @@ class CacheStore:
|
|
|
198
190
|
registry_entry: CachedRegistryEntry,
|
|
199
191
|
key: str,
|
|
200
192
|
value: Any,
|
|
193
|
+
error: Optional[Exception] = None,
|
|
201
194
|
pin: bool = False,
|
|
202
195
|
):
|
|
203
196
|
"""
|
|
@@ -220,11 +213,12 @@ class CacheStore:
|
|
|
220
213
|
|
|
221
214
|
prev_value = await registry_store.get(key)
|
|
222
215
|
|
|
223
|
-
# If
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
216
|
+
# If previous value was a PendingValue, resolve it
|
|
217
|
+
if isinstance(prev_value, PendingValue):
|
|
218
|
+
if error is not None:
|
|
219
|
+
prev_value.error(error)
|
|
220
|
+
else:
|
|
221
|
+
prev_value.resolve(value)
|
|
228
222
|
|
|
229
223
|
# Update size
|
|
230
224
|
self._update_size(prev_value, value)
|
|
@@ -234,6 +228,15 @@ class CacheStore:
|
|
|
234
228
|
|
|
235
229
|
return value
|
|
236
230
|
|
|
231
|
+
async def set_pending(self, registry_entry: CachedRegistryEntry, key: str):
|
|
232
|
+
"""
|
|
233
|
+
Set a pending value for the given registry entry and cache key.
|
|
234
|
+
|
|
235
|
+
:param registry_entry: The registry entry to store the value for.
|
|
236
|
+
:param key: The key of the entry to set.
|
|
237
|
+
"""
|
|
238
|
+
return await self.set(registry_entry, key, PendingValue())
|
|
239
|
+
|
|
237
240
|
async def clear(self):
|
|
238
241
|
"""
|
|
239
242
|
Empty all stores.
|
|
@@ -42,21 +42,18 @@ class KeepAllCache(CacheStoreImpl[KeepAllCachePolicy]):
|
|
|
42
42
|
del self.cache[key]
|
|
43
43
|
return entry
|
|
44
44
|
|
|
45
|
-
async def get(self, key: str, unpin: bool = False
|
|
45
|
+
async def get(self, key: str, unpin: bool = False) -> Optional[Any]:
|
|
46
46
|
"""
|
|
47
47
|
Retrieve a value from the cache.
|
|
48
48
|
|
|
49
49
|
:param key: The key of the value to retrieve.
|
|
50
50
|
:param unpin: This parameter is ignored in KeepAllCache as entries are never evicted.
|
|
51
|
-
:param raise_for_missing: If true, an exception will be raised if the entry is not found
|
|
52
51
|
:return: The value associated with the key, or None if the key is not in the cache.
|
|
53
52
|
"""
|
|
54
53
|
async with self.lock:
|
|
55
54
|
entry = self.cache.get(key)
|
|
56
55
|
|
|
57
56
|
if entry is None:
|
|
58
|
-
if raise_for_missing:
|
|
59
|
-
raise KeyError(f'No cache entry found for {key}')
|
|
60
57
|
return None
|
|
61
58
|
|
|
62
59
|
if unpin:
|
|
@@ -88,13 +88,12 @@ class LRUCache(CacheStoreImpl[LruCachePolicy]):
|
|
|
88
88
|
self.cache.pop(key, None)
|
|
89
89
|
return node.value
|
|
90
90
|
|
|
91
|
-
async def get(self, key: str, unpin: bool = False
|
|
91
|
+
async def get(self, key: str, unpin: bool = False) -> Optional[Any]:
|
|
92
92
|
"""
|
|
93
93
|
Retrieve a value from the cache.
|
|
94
94
|
|
|
95
95
|
:param key: The key of the value to retrieve.
|
|
96
96
|
:param unpin: If true, the entry will be unpinned if it is pinned.
|
|
97
|
-
:param raise_for_missing: If true, an exception will be raised if the entry is not found
|
|
98
97
|
:return: The value associated with the key, or None if the key is not in the cache.
|
|
99
98
|
"""
|
|
100
99
|
async with self.lock:
|
|
@@ -104,9 +103,6 @@ class LRUCache(CacheStoreImpl[LruCachePolicy]):
|
|
|
104
103
|
node.pin = False
|
|
105
104
|
self._move_to_front(node)
|
|
106
105
|
return node.value
|
|
107
|
-
|
|
108
|
-
if raise_for_missing:
|
|
109
|
-
raise KeyError(f'No cache entry found for {key}')
|
|
110
106
|
return None
|
|
111
107
|
|
|
112
108
|
async def set(self, key: str, value: Any, pin: bool = False) -> None:
|
|
@@ -52,13 +52,12 @@ class TTLCache(CacheStoreImpl[TTLCachePolicy]):
|
|
|
52
52
|
_, key = heapq.heappop(self.expiration_heap)
|
|
53
53
|
self.unpinned_cache.pop(key, None)
|
|
54
54
|
|
|
55
|
-
async def get(self, key: str, unpin: bool = False
|
|
55
|
+
async def get(self, key: str, unpin: bool = False) -> Any:
|
|
56
56
|
"""
|
|
57
57
|
Retrieve a value from the cache.
|
|
58
58
|
|
|
59
59
|
:param key: The key of the value to retrieve.
|
|
60
60
|
:param unpin: If true, the entry will be unpinned if it is pinned.
|
|
61
|
-
:param raise_for_missing: If true, an exception will be raised if the entry is not found
|
|
62
61
|
:return: The value associated with the key, or None if the key is not in the cache.
|
|
63
62
|
"""
|
|
64
63
|
async with self.lock:
|
|
@@ -76,8 +75,6 @@ class TTLCache(CacheStoreImpl[TTLCachePolicy]):
|
|
|
76
75
|
elif key in self.unpinned_cache:
|
|
77
76
|
return self.unpinned_cache[key].value
|
|
78
77
|
|
|
79
|
-
if raise_for_missing:
|
|
80
|
-
raise KeyError(f'No cache entry found for {key}')
|
|
81
78
|
return None
|
|
82
79
|
|
|
83
80
|
async def set(self, key: str, value: Any, pin: bool = False) -> None:
|
dara/core/internal/cgroup.py
CHANGED
|
@@ -22,7 +22,7 @@ import sys
|
|
|
22
22
|
|
|
23
23
|
from dara.core.logging import dev_logger, eng_logger
|
|
24
24
|
|
|
25
|
-
CGROUP_V2_INDICATOR_PATH = '/sys/fs/cgroup/cgroup.controllers'
|
|
25
|
+
CGROUP_V2_INDICATOR_PATH = '/sys/fs/cgroup/cgroup.controllers' # Used to determine whether we're using cgroupv2
|
|
26
26
|
|
|
27
27
|
CGROUP_V1_MEM_PATH = '/sys/fs/cgroup/memory/memory.limit_in_bytes'
|
|
28
28
|
CGROUP_V2_MEM_PATH = '/sys/fs/cgroup/memory.max'
|
|
@@ -20,25 +20,36 @@ from typing import Any, List, Literal, Optional, Union
|
|
|
20
20
|
from typing_extensions import TypedDict, TypeGuard
|
|
21
21
|
|
|
22
22
|
from dara.core.base_definitions import BaseTask, PendingTask
|
|
23
|
-
from dara.core.interactivity import DerivedVariable
|
|
24
|
-
from dara.core.interactivity.
|
|
23
|
+
from dara.core.interactivity import DataVariable, DerivedDataVariable, DerivedVariable
|
|
24
|
+
from dara.core.interactivity.filtering import FilterQuery
|
|
25
25
|
from dara.core.internal.cache_store import CacheStore
|
|
26
|
+
from dara.core.internal.pandas_utils import remove_index
|
|
26
27
|
from dara.core.internal.registry_lookup import RegistryLookup
|
|
27
28
|
from dara.core.internal.tasks import TaskManager
|
|
28
29
|
from dara.core.logging import dev_logger
|
|
29
30
|
|
|
30
31
|
|
|
31
32
|
class ResolvedDerivedVariable(TypedDict):
|
|
33
|
+
deps: List[int]
|
|
32
34
|
type: Literal['derived']
|
|
33
35
|
uid: str
|
|
34
36
|
values: List[Any]
|
|
35
|
-
|
|
37
|
+
force: bool
|
|
36
38
|
|
|
37
39
|
|
|
38
|
-
class
|
|
39
|
-
|
|
40
|
+
class ResolvedDerivedDataVariable(TypedDict):
|
|
41
|
+
deps: List[int]
|
|
42
|
+
type: Literal['derived-data']
|
|
43
|
+
uid: str
|
|
44
|
+
values: List[Any]
|
|
45
|
+
filters: Optional[Union[FilterQuery, dict]]
|
|
46
|
+
force: bool
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class ResolvedDataVariable(TypedDict):
|
|
50
|
+
filters: Optional[Union[FilterQuery, dict]]
|
|
51
|
+
type: Literal['data']
|
|
40
52
|
uid: str
|
|
41
|
-
sequence_number: int
|
|
42
53
|
|
|
43
54
|
|
|
44
55
|
class ResolvedSwitchVariable(TypedDict):
|
|
@@ -53,47 +64,21 @@ def is_resolved_derived_variable(obj: Any) -> TypeGuard[ResolvedDerivedVariable]
|
|
|
53
64
|
return isinstance(obj, dict) and 'uid' in obj and obj.get('type') == 'derived'
|
|
54
65
|
|
|
55
66
|
|
|
56
|
-
def
|
|
57
|
-
return isinstance(obj, dict) and 'uid' in obj and obj.get('type') == '
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
def is_resolved_switch_variable(obj: Any) -> TypeGuard[ResolvedSwitchVariable]:
|
|
61
|
-
return isinstance(obj, dict) and 'uid' in obj and obj.get('type') == 'switch'
|
|
67
|
+
def is_resolved_derived_data_variable(obj: Any) -> TypeGuard[ResolvedDerivedDataVariable]:
|
|
68
|
+
return isinstance(obj, dict) and 'uid' in obj and obj.get('type') == 'derived-data'
|
|
62
69
|
|
|
63
70
|
|
|
64
|
-
def
|
|
65
|
-
|
|
66
|
-
Whether a value is a Derived(Data)Variable with a force_key or any of its values are forced
|
|
67
|
-
"""
|
|
68
|
-
if not is_resolved_derived_variable(value):
|
|
69
|
-
return False
|
|
71
|
+
def is_resolved_data_variable(obj: Any) -> TypeGuard[ResolvedDataVariable]:
|
|
72
|
+
return isinstance(obj, dict) and 'uid' in obj and obj.get('type') == 'data'
|
|
70
73
|
|
|
71
|
-
return value.get('force_key') is not None or any(is_forced(v) for v in value.get('values', []))
|
|
72
74
|
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
"""
|
|
76
|
-
Clean an argument to a value to remove force keys
|
|
77
|
-
"""
|
|
78
|
-
if value is None:
|
|
79
|
-
return value
|
|
80
|
-
|
|
81
|
-
if isinstance(value, dict):
|
|
82
|
-
# clone the dict to avoid mutating the original
|
|
83
|
-
value = value.copy()
|
|
84
|
-
# Remove force key from the value
|
|
85
|
-
value.pop('force_key', None)
|
|
86
|
-
return {k: clean_force_key(v) for k, v in value.items()}
|
|
87
|
-
if isinstance(value, list):
|
|
88
|
-
return [clean_force_key(v) for v in value]
|
|
89
|
-
return value
|
|
75
|
+
def is_resolved_switch_variable(obj: Any) -> TypeGuard[ResolvedSwitchVariable]:
|
|
76
|
+
return isinstance(obj, dict) and 'uid' in obj and obj.get('type') == 'switch'
|
|
90
77
|
|
|
91
78
|
|
|
92
79
|
async def resolve_dependency(
|
|
93
80
|
entry: Union[
|
|
94
|
-
ResolvedDerivedVariable,
|
|
95
|
-
ResolvedSwitchVariable,
|
|
96
|
-
Any,
|
|
81
|
+
ResolvedDerivedDataVariable, ResolvedDataVariable, ResolvedDerivedVariable, ResolvedSwitchVariable, Any
|
|
97
82
|
],
|
|
98
83
|
store: CacheStore,
|
|
99
84
|
task_mgr: TaskManager,
|
|
@@ -106,11 +91,14 @@ async def resolve_dependency(
|
|
|
106
91
|
:param store: store instance
|
|
107
92
|
:param task_mgr: task manager instance
|
|
108
93
|
"""
|
|
94
|
+
if is_resolved_derived_data_variable(entry):
|
|
95
|
+
return await _resolve_derived_data_var(entry, store, task_mgr)
|
|
96
|
+
|
|
109
97
|
if is_resolved_derived_variable(entry):
|
|
110
98
|
return await _resolve_derived_var(entry, store, task_mgr)
|
|
111
99
|
|
|
112
|
-
if
|
|
113
|
-
return await
|
|
100
|
+
if is_resolved_data_variable(entry):
|
|
101
|
+
return await _resolve_data_var(entry, store)
|
|
114
102
|
|
|
115
103
|
if is_resolved_switch_variable(entry):
|
|
116
104
|
return await _resolve_switch_var(entry, store, task_mgr)
|
|
@@ -118,10 +106,34 @@ async def resolve_dependency(
|
|
|
118
106
|
return entry
|
|
119
107
|
|
|
120
108
|
|
|
109
|
+
async def _resolve_derived_data_var(entry: ResolvedDerivedDataVariable, store: CacheStore, task_mgr: TaskManager):
|
|
110
|
+
"""
|
|
111
|
+
Resolve a derived data variable from the registry
|
|
112
|
+
|
|
113
|
+
:param entry: derived data variable entry
|
|
114
|
+
:param store: store instance to use for caching
|
|
115
|
+
:param task_mgr: task manager instance
|
|
116
|
+
"""
|
|
117
|
+
from dara.core.internal.registries import (
|
|
118
|
+
data_variable_registry,
|
|
119
|
+
derived_variable_registry,
|
|
120
|
+
utils_registry,
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
registry_mgr: RegistryLookup = utils_registry.get('RegistryLookup')
|
|
124
|
+
dv_var = await registry_mgr.get(derived_variable_registry, str(entry.get('uid')))
|
|
125
|
+
data_var = await registry_mgr.get(data_variable_registry, str(entry.get('uid')))
|
|
126
|
+
|
|
127
|
+
input_values: List[Any] = entry.get('values', [])
|
|
128
|
+
|
|
129
|
+
result = await DerivedDataVariable.resolve_value(
|
|
130
|
+
data_var, dv_var, store, task_mgr, input_values, entry.get('force', False), entry.get('filters', None)
|
|
131
|
+
)
|
|
132
|
+
return remove_index(result)
|
|
133
|
+
|
|
134
|
+
|
|
121
135
|
async def _resolve_derived_var(
|
|
122
|
-
derived_variable_entry: ResolvedDerivedVariable,
|
|
123
|
-
store: CacheStore,
|
|
124
|
-
task_mgr: TaskManager,
|
|
136
|
+
derived_variable_entry: ResolvedDerivedVariable, store: CacheStore, task_mgr: TaskManager
|
|
125
137
|
):
|
|
126
138
|
"""
|
|
127
139
|
Resolve a derived variable from the registry and get it's new value based on the dynamic variable mapping passed
|
|
@@ -137,27 +149,25 @@ async def _resolve_derived_var(
|
|
|
137
149
|
var = await registry_mgr.get(derived_variable_registry, str(derived_variable_entry.get('uid')))
|
|
138
150
|
input_values: List[Any] = derived_variable_entry.get('values', [])
|
|
139
151
|
result = await DerivedVariable.get_value(
|
|
140
|
-
|
|
141
|
-
store=store,
|
|
142
|
-
task_mgr=task_mgr,
|
|
143
|
-
args=input_values,
|
|
144
|
-
force_key=derived_variable_entry.get('force_key'),
|
|
152
|
+
var, store, task_mgr, input_values, derived_variable_entry.get('force', False)
|
|
145
153
|
)
|
|
146
154
|
return result['value']
|
|
147
155
|
|
|
148
156
|
|
|
149
|
-
async def
|
|
157
|
+
async def _resolve_data_var(data_variable_entry: ResolvedDataVariable, store: CacheStore):
|
|
150
158
|
"""
|
|
151
|
-
Resolve a
|
|
159
|
+
Resolve a data variable from the registry and get it's new value based on the dynamic variable mapping passed
|
|
160
|
+
in.
|
|
152
161
|
|
|
153
|
-
:param
|
|
162
|
+
:param data_variable_entry: data var entry
|
|
154
163
|
:param store: the store instance to use for caching
|
|
155
164
|
"""
|
|
156
|
-
from dara.core.internal.registries import
|
|
165
|
+
from dara.core.internal.registries import data_variable_registry, utils_registry
|
|
157
166
|
|
|
158
167
|
registry_mgr: RegistryLookup = utils_registry.get('RegistryLookup')
|
|
159
|
-
|
|
160
|
-
|
|
168
|
+
var = await registry_mgr.get(data_variable_registry, str(data_variable_entry.get('uid')))
|
|
169
|
+
result = await DataVariable.get_value(var, store, data_variable_entry.get('filters', None))
|
|
170
|
+
return remove_index(result)
|
|
161
171
|
|
|
162
172
|
|
|
163
173
|
def _normalize_lookup_key(value: Any) -> str:
|
|
@@ -218,11 +228,7 @@ def _evaluate_condition(condition: dict) -> bool:
|
|
|
218
228
|
raise ValueError(f'Unknown condition operator: {operator}')
|
|
219
229
|
|
|
220
230
|
|
|
221
|
-
async def _resolve_switch_var(
|
|
222
|
-
switch_variable_entry: ResolvedSwitchVariable,
|
|
223
|
-
store: CacheStore,
|
|
224
|
-
task_mgr: TaskManager,
|
|
225
|
-
):
|
|
231
|
+
async def _resolve_switch_var(switch_variable_entry: ResolvedSwitchVariable, store: CacheStore, task_mgr: TaskManager):
|
|
226
232
|
"""
|
|
227
233
|
Resolve a switch variable by evaluating its constituent parts and returning the appropriate value.
|
|
228
234
|
|
dara/core/internal/devtools.py
CHANGED
|
@@ -21,26 +21,22 @@ import sys
|
|
|
21
21
|
import traceback
|
|
22
22
|
from contextlib import contextmanager
|
|
23
23
|
from datetime import datetime
|
|
24
|
-
from typing import Optional
|
|
25
24
|
|
|
26
25
|
from dara.core.internal.websocket import WebsocketManager
|
|
27
26
|
from dara.core.logging import eng_logger
|
|
28
27
|
|
|
29
28
|
|
|
30
|
-
def print_stacktrace(
|
|
29
|
+
def print_stacktrace():
|
|
31
30
|
"""
|
|
32
31
|
Prints out the current stack trace. Will also extract any exceptions and print them at the end.
|
|
33
32
|
"""
|
|
34
|
-
if err is not None:
|
|
35
|
-
return ''.join(traceback.format_exception(type(err), err, err.__traceback__))
|
|
36
|
-
|
|
37
33
|
exc = sys.exc_info()[0]
|
|
38
34
|
stack = traceback.extract_stack()[:-1]
|
|
39
35
|
|
|
40
36
|
trc = 'Traceback (most recent call last):\n'
|
|
41
37
|
stackstr = trc + ''.join(traceback.format_list(stack))
|
|
42
38
|
if exc is not None:
|
|
43
|
-
stackstr += ' ' + traceback.format_exc().lstrip(trc)
|
|
39
|
+
stackstr += ' ' + traceback.format_exc().lstrip(trc) # pylint:disable=bad-str-strip-call
|
|
44
40
|
else:
|
|
45
41
|
stackstr += ' Exception'
|
|
46
42
|
|
|
@@ -56,17 +52,14 @@ def handle_system_exit(error_msg: str):
|
|
|
56
52
|
try:
|
|
57
53
|
yield
|
|
58
54
|
except SystemExit as e:
|
|
59
|
-
raise InterruptedError(error_msg)
|
|
55
|
+
raise InterruptedError(error_msg).with_traceback(e.__traceback__)
|
|
60
56
|
|
|
61
57
|
|
|
62
|
-
def get_error_for_channel(
|
|
58
|
+
def get_error_for_channel() -> dict:
|
|
63
59
|
"""
|
|
64
60
|
Get error from current stacktrace to send to the client
|
|
65
61
|
"""
|
|
66
|
-
return {
|
|
67
|
-
'error': print_stacktrace(err),
|
|
68
|
-
'time': str(datetime.now()),
|
|
69
|
-
}
|
|
62
|
+
return {'error': print_stacktrace(), 'time': str(datetime.now())}
|
|
70
63
|
|
|
71
64
|
|
|
72
65
|
async def send_error_for_session(ws_mgr: WebsocketManager, session_id: str):
|
dara/core/internal/download.py
CHANGED
|
@@ -18,9 +18,7 @@ limitations under the License.
|
|
|
18
18
|
from __future__ import annotations
|
|
19
19
|
|
|
20
20
|
import os
|
|
21
|
-
from
|
|
22
|
-
from contextvars import ContextVar
|
|
23
|
-
from typing import Callable, Optional, Tuple
|
|
21
|
+
from typing import Awaitable, Callable, Optional, Tuple
|
|
24
22
|
from uuid import uuid4
|
|
25
23
|
|
|
26
24
|
import anyio
|
|
@@ -39,13 +37,13 @@ class DownloadDataEntry(BaseModel):
|
|
|
39
37
|
file_path: str
|
|
40
38
|
cleanup_file: bool
|
|
41
39
|
identity_name: Optional[str] = None
|
|
42
|
-
download: Callable[[DownloadDataEntry], Awaitable[Tuple[anyio.AsyncFile, Callable[..., Awaitable]]]]
|
|
40
|
+
download: Callable[['DownloadDataEntry'], Awaitable[Tuple[anyio.AsyncFile, Callable[..., Awaitable]]]]
|
|
43
41
|
"""Handler for getting the file from the entry"""
|
|
44
42
|
|
|
45
43
|
|
|
46
44
|
DownloadRegistryEntry = CachedRegistryEntry(
|
|
47
45
|
uid='_dara_download', cache=Cache.Policy.TTL(ttl=60 * 10)
|
|
48
|
-
)
|
|
46
|
+
) # expire the codes after 10 minutes
|
|
49
47
|
|
|
50
48
|
|
|
51
49
|
async def download(data_entry: DownloadDataEntry) -> Tuple[anyio.AsyncFile, Callable[..., Awaitable]]:
|
|
@@ -74,13 +72,6 @@ async def download(data_entry: DownloadDataEntry) -> Tuple[anyio.AsyncFile, Call
|
|
|
74
72
|
return (async_file, cleanup)
|
|
75
73
|
|
|
76
74
|
|
|
77
|
-
GENERATE_CODE_OVERRIDE = ContextVar[Optional[Callable[[str], str]]]('GENERATE_CODE_OVERRIDE', default=None)
|
|
78
|
-
"""
|
|
79
|
-
Optional context variable which can be used to override the default behaviour of code generation.
|
|
80
|
-
Invoked with the file path to generate a download code for.
|
|
81
|
-
"""
|
|
82
|
-
|
|
83
|
-
|
|
84
75
|
async def generate_download_code(file_path: str, cleanup_file: bool) -> str:
|
|
85
76
|
"""
|
|
86
77
|
Generate a one-time download code for a given dataset.
|
|
@@ -96,7 +87,7 @@ async def generate_download_code(file_path: str, cleanup_file: bool) -> str:
|
|
|
96
87
|
user = USER.get()
|
|
97
88
|
|
|
98
89
|
# Unique download id
|
|
99
|
-
uid =
|
|
90
|
+
uid = str(uuid4())
|
|
100
91
|
|
|
101
92
|
# Put it in the store under the registry entry with TTL configured
|
|
102
93
|
await store.set(
|
|
@@ -14,13 +14,13 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
14
14
|
See the License for the specific language governing permissions and
|
|
15
15
|
limitations under the License.
|
|
16
16
|
"""
|
|
17
|
-
|
|
18
|
-
from collections.abc import MutableMapping
|
|
17
|
+
# pylint: disable=unnecessary-lambda
|
|
19
18
|
from inspect import Parameter, isclass
|
|
20
19
|
from typing import (
|
|
21
20
|
Any,
|
|
22
21
|
Callable,
|
|
23
22
|
Dict,
|
|
23
|
+
MutableMapping,
|
|
24
24
|
Optional,
|
|
25
25
|
Type,
|
|
26
26
|
Union,
|
|
@@ -99,11 +99,11 @@ def _tuple_key_deserialize(d):
|
|
|
99
99
|
if isinstance(key, str) and key.startswith('__tuple__'):
|
|
100
100
|
key_list = []
|
|
101
101
|
for each in key[10:-1].split(', '):
|
|
102
|
-
if (each.startswith("'") and each.endswith("'")) or (each.startswith('"') and each.endswith('"')):
|
|
102
|
+
if (each.startswith("'") and each.endswith(("'"))) or (each.startswith('"') and each.endswith(('"'))):
|
|
103
103
|
key_list.append(each[1:-1])
|
|
104
104
|
else:
|
|
105
105
|
key_list.append(each)
|
|
106
|
-
encoded_key = tuple(key_list)
|
|
106
|
+
encoded_key = encoded_key = tuple(key_list)
|
|
107
107
|
else:
|
|
108
108
|
encoded_key = key
|
|
109
109
|
|
|
@@ -112,7 +112,7 @@ def _tuple_key_deserialize(d):
|
|
|
112
112
|
return encoded_dict
|
|
113
113
|
|
|
114
114
|
|
|
115
|
-
def _df_deserialize(x):
|
|
115
|
+
def _df_deserialize(x): # pylint: disable=inconsistent-return-statements
|
|
116
116
|
"""
|
|
117
117
|
A function to deserialize data into a DataFrame
|
|
118
118
|
|
|
@@ -240,14 +240,14 @@ def deserialize(value: Any, typ: Optional[Type]):
|
|
|
240
240
|
return value
|
|
241
241
|
|
|
242
242
|
# Already matches type
|
|
243
|
-
if type(value)
|
|
243
|
+
if type(value) == typ:
|
|
244
244
|
return value
|
|
245
245
|
|
|
246
246
|
# Handle Optional[foo] / Union[foo, None] -> call deserialize(value, foo)
|
|
247
247
|
if get_origin(typ) == Union:
|
|
248
248
|
args = get_args(typ)
|
|
249
249
|
if len(args) == 2 and type(None) in args:
|
|
250
|
-
not_none_arg = args[0] if args[0]
|
|
250
|
+
not_none_arg = args[0] if args[0] != type(None) else args[1]
|
|
251
251
|
return deserialize(value, not_none_arg)
|
|
252
252
|
|
|
253
253
|
try:
|
|
@@ -14,13 +14,11 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
14
14
|
See the License for the specific language governing permissions and
|
|
15
15
|
limitations under the License.
|
|
16
16
|
"""
|
|
17
|
-
|
|
18
17
|
from __future__ import annotations
|
|
19
18
|
|
|
20
19
|
import asyncio
|
|
21
|
-
from collections.abc import Mapping
|
|
22
20
|
from contextvars import ContextVar
|
|
23
|
-
from typing import Any, Callable, Optional, Union
|
|
21
|
+
from typing import Any, Callable, Mapping, Optional, Union
|
|
24
22
|
|
|
25
23
|
import anyio
|
|
26
24
|
|
|
@@ -32,7 +30,11 @@ from dara.core.interactivity.actions import (
|
|
|
32
30
|
ActionImpl,
|
|
33
31
|
)
|
|
34
32
|
from dara.core.internal.cache_store import CacheStore
|
|
35
|
-
from dara.core.internal.dependency_resolution import
|
|
33
|
+
from dara.core.internal.dependency_resolution import (
|
|
34
|
+
is_resolved_derived_data_variable,
|
|
35
|
+
is_resolved_derived_variable,
|
|
36
|
+
resolve_dependency,
|
|
37
|
+
)
|
|
36
38
|
from dara.core.internal.encoder_registry import deserialize
|
|
37
39
|
from dara.core.internal.tasks import MetaTask, TaskManager
|
|
38
40
|
from dara.core.internal.utils import run_user_handler
|
|
@@ -143,15 +145,15 @@ async def execute_action(
|
|
|
143
145
|
if values is not None:
|
|
144
146
|
annotations = action.__annotations__
|
|
145
147
|
|
|
146
|
-
|
|
148
|
+
for key, value in values.items():
|
|
149
|
+
# Override `force` property to be false
|
|
150
|
+
if is_resolved_derived_variable(value) or is_resolved_derived_data_variable(value):
|
|
151
|
+
value['force'] = False
|
|
152
|
+
|
|
147
153
|
typ = annotations.get(key)
|
|
148
|
-
val = await resolve_dependency(
|
|
154
|
+
val = await resolve_dependency(value, store, task_mgr)
|
|
149
155
|
resolved_kwargs[key] = deserialize(val, typ)
|
|
150
156
|
|
|
151
|
-
async with anyio.create_task_group() as tg:
|
|
152
|
-
for key, value in values.items():
|
|
153
|
-
tg.start_soon(_resolve_kwarg, value, key)
|
|
154
|
-
|
|
155
157
|
# Merge resolved dynamic kwargs with static kwargs received
|
|
156
158
|
resolved_kwargs = {**resolved_kwargs, **static_kwargs}
|
|
157
159
|
|
|
@@ -175,11 +177,9 @@ async def execute_action(
|
|
|
175
177
|
|
|
176
178
|
# Note: no associated registry entry, the result are not persisted in cache
|
|
177
179
|
# Return a metatask which, when all dependencies are ready, will stream the action results to the frontend
|
|
178
|
-
|
|
180
|
+
return MetaTask(
|
|
179
181
|
process_result=_stream_action, args=[action, ctx], kwargs=resolved_kwargs, notify_channels=notify_channels
|
|
180
182
|
)
|
|
181
|
-
task_mgr.register_task(meta_task)
|
|
182
|
-
return meta_task
|
|
183
183
|
|
|
184
184
|
# No tasks - run directly as an asyncio task and return the execution id
|
|
185
185
|
# Originally used to use FastAPI BackgroundTasks, but these ended up causing a blocking behavior that blocked some
|
dara/core/internal/hashing.py
CHANGED
|
@@ -31,6 +31,8 @@ def hash_object(obj: Union[BaseModel, dict, None]):
|
|
|
31
31
|
if isinstance(obj, BaseModel):
|
|
32
32
|
obj = obj.model_dump()
|
|
33
33
|
|
|
34
|
-
filter_hash = hashlib.sha1(
|
|
34
|
+
filter_hash = hashlib.sha1(
|
|
35
|
+
usedforsecurity=False
|
|
36
|
+
) # nosec B303 # we don't use this for security purposes just as a cache key
|
|
35
37
|
filter_hash.update(json.dumps(obj or {}, sort_keys=True).encode())
|
|
36
38
|
return filter_hash.hexdigest()
|
|
@@ -93,9 +93,10 @@ def run_discovery(
|
|
|
93
93
|
# If module root is passed through, use it
|
|
94
94
|
if 'module_root' in kwargs:
|
|
95
95
|
root = kwargs.get('module_root')
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
96
|
+
else:
|
|
97
|
+
# Try to infer from module_name
|
|
98
|
+
if module_name is not None:
|
|
99
|
+
root = module_name.split('.')[0]
|
|
99
100
|
|
|
100
101
|
for k, v in global_symbols.items():
|
|
101
102
|
# Ignore already encountered functions
|