lionherd-core 1.0.0a3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lionherd_core/__init__.py +84 -0
- lionherd_core/base/__init__.py +30 -0
- lionherd_core/base/_utils.py +295 -0
- lionherd_core/base/broadcaster.py +128 -0
- lionherd_core/base/element.py +300 -0
- lionherd_core/base/event.py +322 -0
- lionherd_core/base/eventbus.py +112 -0
- lionherd_core/base/flow.py +236 -0
- lionherd_core/base/graph.py +616 -0
- lionherd_core/base/node.py +212 -0
- lionherd_core/base/pile.py +811 -0
- lionherd_core/base/progression.py +261 -0
- lionherd_core/errors.py +104 -0
- lionherd_core/libs/__init__.py +2 -0
- lionherd_core/libs/concurrency/__init__.py +60 -0
- lionherd_core/libs/concurrency/_cancel.py +85 -0
- lionherd_core/libs/concurrency/_errors.py +80 -0
- lionherd_core/libs/concurrency/_patterns.py +238 -0
- lionherd_core/libs/concurrency/_primitives.py +253 -0
- lionherd_core/libs/concurrency/_priority_queue.py +135 -0
- lionherd_core/libs/concurrency/_resource_tracker.py +66 -0
- lionherd_core/libs/concurrency/_task.py +58 -0
- lionherd_core/libs/concurrency/_utils.py +61 -0
- lionherd_core/libs/schema_handlers/__init__.py +35 -0
- lionherd_core/libs/schema_handlers/_function_call_parser.py +122 -0
- lionherd_core/libs/schema_handlers/_minimal_yaml.py +88 -0
- lionherd_core/libs/schema_handlers/_schema_to_model.py +251 -0
- lionherd_core/libs/schema_handlers/_typescript.py +153 -0
- lionherd_core/libs/string_handlers/__init__.py +15 -0
- lionherd_core/libs/string_handlers/_extract_json.py +65 -0
- lionherd_core/libs/string_handlers/_fuzzy_json.py +103 -0
- lionherd_core/libs/string_handlers/_string_similarity.py +347 -0
- lionherd_core/libs/string_handlers/_to_num.py +63 -0
- lionherd_core/ln/__init__.py +45 -0
- lionherd_core/ln/_async_call.py +314 -0
- lionherd_core/ln/_fuzzy_match.py +166 -0
- lionherd_core/ln/_fuzzy_validate.py +151 -0
- lionherd_core/ln/_hash.py +141 -0
- lionherd_core/ln/_json_dump.py +347 -0
- lionherd_core/ln/_list_call.py +110 -0
- lionherd_core/ln/_to_dict.py +373 -0
- lionherd_core/ln/_to_list.py +190 -0
- lionherd_core/ln/_utils.py +156 -0
- lionherd_core/lndl/__init__.py +62 -0
- lionherd_core/lndl/errors.py +30 -0
- lionherd_core/lndl/fuzzy.py +321 -0
- lionherd_core/lndl/parser.py +427 -0
- lionherd_core/lndl/prompt.py +137 -0
- lionherd_core/lndl/resolver.py +323 -0
- lionherd_core/lndl/types.py +287 -0
- lionherd_core/protocols.py +181 -0
- lionherd_core/py.typed +0 -0
- lionherd_core/types/__init__.py +46 -0
- lionherd_core/types/_sentinel.py +131 -0
- lionherd_core/types/base.py +341 -0
- lionherd_core/types/operable.py +133 -0
- lionherd_core/types/spec.py +313 -0
- lionherd_core/types/spec_adapters/__init__.py +10 -0
- lionherd_core/types/spec_adapters/_protocol.py +125 -0
- lionherd_core/types/spec_adapters/pydantic_field.py +177 -0
- lionherd_core-1.0.0a3.dist-info/METADATA +502 -0
- lionherd_core-1.0.0a3.dist-info/RECORD +64 -0
- lionherd_core-1.0.0a3.dist-info/WHEEL +4 -0
- lionherd_core-1.0.0a3.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,811 @@
|
|
|
1
|
+
# Copyright (c) 2025, HaiyangLi <quantocean.li at gmail dot com>
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import contextlib
|
|
7
|
+
import threading
|
|
8
|
+
from collections.abc import Callable, Iterator
|
|
9
|
+
from typing import Any, Generic, Literal, TypeVar, overload
|
|
10
|
+
from uuid import UUID
|
|
11
|
+
|
|
12
|
+
from pydantic import Field, PrivateAttr, field_serializer, field_validator
|
|
13
|
+
from pydapter import (
|
|
14
|
+
Adaptable as PydapterAdaptable,
|
|
15
|
+
AsyncAdaptable as PydapterAsyncAdaptable,
|
|
16
|
+
)
|
|
17
|
+
from typing_extensions import override
|
|
18
|
+
|
|
19
|
+
from ..libs.concurrency import Lock as AsyncLock
|
|
20
|
+
from ..protocols import (
|
|
21
|
+
Adaptable,
|
|
22
|
+
AsyncAdaptable,
|
|
23
|
+
Containable,
|
|
24
|
+
Deserializable,
|
|
25
|
+
Serializable,
|
|
26
|
+
implements,
|
|
27
|
+
)
|
|
28
|
+
from ._utils import (
|
|
29
|
+
async_synchronized,
|
|
30
|
+
extract_types,
|
|
31
|
+
load_type_from_string,
|
|
32
|
+
synchronized,
|
|
33
|
+
to_uuid,
|
|
34
|
+
)
|
|
35
|
+
from .element import Element
|
|
36
|
+
from .progression import Progression
|
|
37
|
+
|
|
38
|
+
__all__ = ("Pile",)
|
|
39
|
+
|
|
40
|
+
T = TypeVar("T", bound=Element)
|
|
41
|
+
|
|
42
|
+
PILE_REGISTRY: dict[str, type[Pile]] = {}
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@implements(Containable, Adaptable, AsyncAdaptable, Serializable, Deserializable)
|
|
46
|
+
class Pile(Element, PydapterAdaptable, PydapterAsyncAdaptable, Generic[T]):
|
|
47
|
+
"""Thread-safe typed collection with rich query interface.
|
|
48
|
+
|
|
49
|
+
Type-dispatched __getitem__: pile[uuid], pile[int/slice], pile[progression], pile[callable].
|
|
50
|
+
|
|
51
|
+
Args:
|
|
52
|
+
items: Initial items
|
|
53
|
+
item_type: Type(s) for validation (single/set/list/Union)
|
|
54
|
+
strict_type: Enforce exact type match (no subclasses)
|
|
55
|
+
|
|
56
|
+
Adapter Registration (Rust-like isolated pattern):
|
|
57
|
+
Each Pile subclass has its own independent adapter registry. No auto-registration.
|
|
58
|
+
Must explicitly register adapters on each class that needs them:
|
|
59
|
+
|
|
60
|
+
```python
|
|
61
|
+
from pydapter.adapters import TomlAdapter
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class CustomPile(Pile):
|
|
65
|
+
pass
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
# Must register explicitly (no inheritance from parent)
|
|
69
|
+
CustomPile.register_adapter(TomlAdapter)
|
|
70
|
+
custom_pile.adapt_to("toml") # Now works
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
This prevents adapter pollution and ensures explicit control per class.
|
|
74
|
+
"""
|
|
75
|
+
|
|
76
|
+
# Private internal state - excluded from serialization
|
|
77
|
+
_items: dict[UUID, T] = PrivateAttr(default_factory=dict)
|
|
78
|
+
_progression: Progression = PrivateAttr(default_factory=Progression)
|
|
79
|
+
_lock: threading.RLock = PrivateAttr(default_factory=threading.RLock)
|
|
80
|
+
_async_lock: AsyncLock = PrivateAttr(default_factory=AsyncLock)
|
|
81
|
+
|
|
82
|
+
# Properties for internal access (return immutable views)
|
|
83
|
+
@property
|
|
84
|
+
def items(self):
|
|
85
|
+
"""Items as read-only mapping view."""
|
|
86
|
+
from types import MappingProxyType
|
|
87
|
+
|
|
88
|
+
return MappingProxyType(self._items)
|
|
89
|
+
|
|
90
|
+
@property
|
|
91
|
+
def progression(self) -> Progression:
|
|
92
|
+
"""Progression order as read-only copy."""
|
|
93
|
+
# Return copy to prevent external modification
|
|
94
|
+
return Progression(order=list(self._progression.order), name=self._progression.name)
|
|
95
|
+
|
|
96
|
+
# Type validation config
|
|
97
|
+
item_type: set[type] | None = Field(
|
|
98
|
+
default=None,
|
|
99
|
+
description="Set of allowed types for validation (None = any Element subclass)",
|
|
100
|
+
)
|
|
101
|
+
strict_type: bool = Field(
|
|
102
|
+
default=False,
|
|
103
|
+
description="If True, enforce exact type match (no subclasses allowed)",
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
@field_validator("item_type", mode="before")
|
|
107
|
+
@classmethod
|
|
108
|
+
def _normalize_item_type(cls, v: Any) -> set[type] | None:
|
|
109
|
+
"""Normalize item_type to set[type] (handles deserialization and runtime Union/list/set)."""
|
|
110
|
+
if v is None:
|
|
111
|
+
return None
|
|
112
|
+
|
|
113
|
+
# Deserialization case: ["module.ClassName", ...] → {type, ...}
|
|
114
|
+
if isinstance(v, list) and v and isinstance(v[0], str):
|
|
115
|
+
return {load_type_from_string(type_str) for type_str in v}
|
|
116
|
+
|
|
117
|
+
# Runtime case: Union[A, B] | [A, B] | {A, B} | A → {A, B}
|
|
118
|
+
return extract_types(v)
|
|
119
|
+
|
|
120
|
+
@override
|
|
121
|
+
def __init__(
|
|
122
|
+
self,
|
|
123
|
+
items: list[T] | None = None,
|
|
124
|
+
item_type: type[T] | set[type] | list[type] | None = None,
|
|
125
|
+
order: list[UUID] | Progression | None = None,
|
|
126
|
+
strict_type: bool = False,
|
|
127
|
+
**kwargs,
|
|
128
|
+
):
|
|
129
|
+
"""Initialize Pile with optional items.
|
|
130
|
+
|
|
131
|
+
Args:
|
|
132
|
+
items: Initial items to add to the pile
|
|
133
|
+
item_type: Type(s) for validation (single type, set, list, or Union)
|
|
134
|
+
order: Order of items (list of UUIDs or Progression instance)
|
|
135
|
+
strict_type: If True, enforce exact type match (no subclasses)
|
|
136
|
+
**kwargs: Additional Element fields (id, created_at, metadata, etc.)
|
|
137
|
+
"""
|
|
138
|
+
# Initialize Pydantic model with fields (pass through **kwargs for mypy)
|
|
139
|
+
super().__init__(**{"item_type": item_type, "strict_type": strict_type, **kwargs})
|
|
140
|
+
|
|
141
|
+
# Add items after initialization (uses _items PrivateAttr)
|
|
142
|
+
if items:
|
|
143
|
+
for item in items:
|
|
144
|
+
self.add(item)
|
|
145
|
+
|
|
146
|
+
# Set custom order if provided (overrides insertion order)
|
|
147
|
+
if order:
|
|
148
|
+
order_list = list(order.order) if isinstance(order, Progression) else order
|
|
149
|
+
|
|
150
|
+
# Validate that all UUIDs in order are in items
|
|
151
|
+
for uid in order_list:
|
|
152
|
+
if uid not in self._items:
|
|
153
|
+
raise ValueError(f"UUID {uid} in order not found in items")
|
|
154
|
+
# Set progression order
|
|
155
|
+
self._progression = Progression(order=order_list)
|
|
156
|
+
|
|
157
|
+
# ==================== Serialization ====================
|
|
158
|
+
|
|
159
|
+
@field_serializer("item_type")
|
|
160
|
+
def _serialize_item_type(self, v: set[type] | None) -> list[str] | None:
|
|
161
|
+
"""Serialize item_type set to list of module paths."""
|
|
162
|
+
if v is None:
|
|
163
|
+
return None
|
|
164
|
+
return [f"{t.__module__}.{t.__name__}" for t in v]
|
|
165
|
+
|
|
166
|
+
@override
|
|
167
|
+
def to_dict(
|
|
168
|
+
self,
|
|
169
|
+
mode: Literal["python", "json", "db"] = "python",
|
|
170
|
+
created_at_format: Literal["datetime", "isoformat", "timestamp"] | None = None,
|
|
171
|
+
meta_key: str | None = None,
|
|
172
|
+
item_meta_key: str | None = None,
|
|
173
|
+
item_created_at_format: Literal["datetime", "isoformat", "timestamp"] | None = None,
|
|
174
|
+
**kwargs: Any,
|
|
175
|
+
) -> dict[str, Any]:
|
|
176
|
+
"""Serialize pile with items in progression order.
|
|
177
|
+
|
|
178
|
+
Args:
|
|
179
|
+
mode: python/json/db
|
|
180
|
+
created_at_format: Timestamp format for Pile
|
|
181
|
+
meta_key: Rename Pile metadata field
|
|
182
|
+
item_meta_key: Pass to each item's to_dict for metadata renaming
|
|
183
|
+
item_created_at_format: Pass to each item's to_dict for timestamp format
|
|
184
|
+
**kwargs: Passed to model_dump()
|
|
185
|
+
"""
|
|
186
|
+
# Get base Element serialization (will handle meta_key renaming)
|
|
187
|
+
data = super().to_dict(
|
|
188
|
+
mode=mode, created_at_format=created_at_format, meta_key=meta_key, **kwargs
|
|
189
|
+
)
|
|
190
|
+
|
|
191
|
+
# Determine the actual metadata key name in the output
|
|
192
|
+
# (will be renamed if meta_key was specified, or "node_metadata" for db mode)
|
|
193
|
+
actual_meta_key = (
|
|
194
|
+
meta_key if meta_key else ("node_metadata" if mode == "db" else "metadata")
|
|
195
|
+
)
|
|
196
|
+
|
|
197
|
+
# Store progression metadata in pile's metadata
|
|
198
|
+
if self._progression.name and actual_meta_key in data:
|
|
199
|
+
if data[actual_meta_key] is None:
|
|
200
|
+
data[actual_meta_key] = {}
|
|
201
|
+
data[actual_meta_key]["progression_name"] = self._progression.name
|
|
202
|
+
|
|
203
|
+
# Serialize items in progression order (progression order is implicit)
|
|
204
|
+
if mode == "python":
|
|
205
|
+
# Python mode: keep objects as-is
|
|
206
|
+
data["items"] = [
|
|
207
|
+
self._items[uid].to_dict(
|
|
208
|
+
mode="python",
|
|
209
|
+
meta_key=item_meta_key,
|
|
210
|
+
created_at_format=item_created_at_format,
|
|
211
|
+
)
|
|
212
|
+
for uid in self._progression
|
|
213
|
+
if uid in self._items
|
|
214
|
+
]
|
|
215
|
+
else:
|
|
216
|
+
# JSON/DB mode: convert to JSON-safe
|
|
217
|
+
data["items"] = [
|
|
218
|
+
self._items[uid].to_dict(
|
|
219
|
+
mode="json",
|
|
220
|
+
meta_key=item_meta_key,
|
|
221
|
+
created_at_format=item_created_at_format,
|
|
222
|
+
)
|
|
223
|
+
for uid in self._progression
|
|
224
|
+
if uid in self._items
|
|
225
|
+
]
|
|
226
|
+
|
|
227
|
+
return data
|
|
228
|
+
|
|
229
|
+
# ==================== Core Operations ====================
|
|
230
|
+
|
|
231
|
+
@synchronized
|
|
232
|
+
def add(self, item: T) -> None:
|
|
233
|
+
"""Add item to pile.
|
|
234
|
+
|
|
235
|
+
Args:
|
|
236
|
+
item: Element to add
|
|
237
|
+
|
|
238
|
+
Raises:
|
|
239
|
+
ValueError: If item already exists or type validation fails
|
|
240
|
+
"""
|
|
241
|
+
self._validate_type(item)
|
|
242
|
+
|
|
243
|
+
if item.id in self._items:
|
|
244
|
+
raise ValueError(f"Item {item.id} already exists in pile")
|
|
245
|
+
|
|
246
|
+
self._items[item.id] = item
|
|
247
|
+
self._progression.append(item.id)
|
|
248
|
+
|
|
249
|
+
@synchronized
|
|
250
|
+
def remove(self, item_id: UUID | str | Element) -> T:
|
|
251
|
+
"""Remove item from pile.
|
|
252
|
+
|
|
253
|
+
Args:
|
|
254
|
+
item_id: Item ID or Element instance
|
|
255
|
+
|
|
256
|
+
Returns:
|
|
257
|
+
Removed item
|
|
258
|
+
|
|
259
|
+
Raises:
|
|
260
|
+
ValueError: If item not found
|
|
261
|
+
"""
|
|
262
|
+
|
|
263
|
+
uid = to_uuid(item_id)
|
|
264
|
+
|
|
265
|
+
if uid not in self._items:
|
|
266
|
+
raise ValueError(f"Item {uid} not found in pile")
|
|
267
|
+
|
|
268
|
+
item = self._items.pop(uid)
|
|
269
|
+
self._progression.remove(uid)
|
|
270
|
+
return item
|
|
271
|
+
|
|
272
|
+
def pop(self, item_id: UUID | str | Element) -> T:
|
|
273
|
+
"""Alias for remove() - pop item from pile."""
|
|
274
|
+
return self.remove(item_id)
|
|
275
|
+
|
|
276
|
+
@synchronized
|
|
277
|
+
def get(self, item_id: UUID | str | Element, default: Any = ...) -> T | None:
|
|
278
|
+
"""Get item by ID with optional default.
|
|
279
|
+
|
|
280
|
+
Args:
|
|
281
|
+
item_id: Item ID or Element instance
|
|
282
|
+
default: Default value if not found
|
|
283
|
+
|
|
284
|
+
Returns:
|
|
285
|
+
Item or default
|
|
286
|
+
|
|
287
|
+
Raises:
|
|
288
|
+
ValueError: If item not found and no default
|
|
289
|
+
"""
|
|
290
|
+
|
|
291
|
+
uid = to_uuid(item_id)
|
|
292
|
+
|
|
293
|
+
if uid not in self._items:
|
|
294
|
+
if default is ...:
|
|
295
|
+
raise ValueError(f"Item {uid} not found in pile")
|
|
296
|
+
return default
|
|
297
|
+
|
|
298
|
+
return self._items[uid]
|
|
299
|
+
|
|
300
|
+
@synchronized
|
|
301
|
+
def update(self, item: T) -> None:
|
|
302
|
+
"""Update existing item.
|
|
303
|
+
|
|
304
|
+
Args:
|
|
305
|
+
item: Updated item (must have same ID)
|
|
306
|
+
|
|
307
|
+
Raises:
|
|
308
|
+
ValueError: If item not found or type validation fails
|
|
309
|
+
"""
|
|
310
|
+
self._validate_type(item)
|
|
311
|
+
|
|
312
|
+
if item.id not in self._items:
|
|
313
|
+
raise ValueError(f"Item {item.id} not found in pile")
|
|
314
|
+
|
|
315
|
+
self._items[item.id] = item
|
|
316
|
+
|
|
317
|
+
@synchronized
|
|
318
|
+
def clear(self) -> None:
|
|
319
|
+
"""Remove all items."""
|
|
320
|
+
self._items.clear()
|
|
321
|
+
self._progression.clear()
|
|
322
|
+
|
|
323
|
+
# ==================== Set-like Operations ====================
|
|
324
|
+
|
|
325
|
+
def include(self, item: T) -> bool:
|
|
326
|
+
"""Include item in pile (idempotent).
|
|
327
|
+
|
|
328
|
+
Returns:
|
|
329
|
+
bool: True if item was added, False if already present
|
|
330
|
+
"""
|
|
331
|
+
if item.id not in self._items:
|
|
332
|
+
self.add(item)
|
|
333
|
+
return True
|
|
334
|
+
return False
|
|
335
|
+
|
|
336
|
+
def exclude(self, item: UUID | str | Element) -> bool:
|
|
337
|
+
"""Exclude item from pile (idempotent).
|
|
338
|
+
|
|
339
|
+
Returns:
|
|
340
|
+
bool: True if item was removed, False if not present
|
|
341
|
+
"""
|
|
342
|
+
|
|
343
|
+
uid = to_uuid(item)
|
|
344
|
+
if uid in self._items:
|
|
345
|
+
self.remove(uid)
|
|
346
|
+
return True
|
|
347
|
+
return False
|
|
348
|
+
|
|
349
|
+
# ==================== Rich __getitem__ (Type Dispatch) ====================
|
|
350
|
+
|
|
351
|
+
@overload
|
|
352
|
+
def __getitem__(self, key: UUID | str) -> T:
|
|
353
|
+
"""Get single item by UUID or string ID."""
|
|
354
|
+
...
|
|
355
|
+
|
|
356
|
+
@overload
|
|
357
|
+
def __getitem__(self, key: Progression) -> Pile[T]:
|
|
358
|
+
"""Filter by progression - returns new Pile."""
|
|
359
|
+
...
|
|
360
|
+
|
|
361
|
+
@overload
|
|
362
|
+
def __getitem__(self, key: int) -> T:
|
|
363
|
+
"""Get item by index."""
|
|
364
|
+
...
|
|
365
|
+
|
|
366
|
+
@overload
|
|
367
|
+
def __getitem__(self, key: slice) -> list[T]:
|
|
368
|
+
"""Get multiple items by slice."""
|
|
369
|
+
...
|
|
370
|
+
|
|
371
|
+
@overload
|
|
372
|
+
def __getitem__(self, key: Callable[[T], bool]) -> Pile[T]:
|
|
373
|
+
"""Filter by function - returns new Pile."""
|
|
374
|
+
...
|
|
375
|
+
|
|
376
|
+
def __getitem__(self, key: Any) -> T | list[T] | Pile[T]:
|
|
377
|
+
"""Type-dispatched query interface.
|
|
378
|
+
|
|
379
|
+
Args:
|
|
380
|
+
key: UUID/str (get by ID), Progression (filter), int/slice (index), or callable (predicate)
|
|
381
|
+
|
|
382
|
+
Returns:
|
|
383
|
+
Element, list[Element], or new Pile (depends on key type)
|
|
384
|
+
|
|
385
|
+
Raises:
|
|
386
|
+
TypeError: If key type is not supported
|
|
387
|
+
ValueError: If item not found
|
|
388
|
+
"""
|
|
389
|
+
# Type 1: UUID/str - Get by ID
|
|
390
|
+
if isinstance(key, (UUID, str)):
|
|
391
|
+
return self.get(key)
|
|
392
|
+
|
|
393
|
+
# Type 2: Progression - Filter by progression (RETURNS NEW PILE!)
|
|
394
|
+
elif isinstance(key, Progression):
|
|
395
|
+
return self._filter_by_progression(key)
|
|
396
|
+
|
|
397
|
+
# Type 3: int - Index access
|
|
398
|
+
elif isinstance(key, int):
|
|
399
|
+
return self._get_by_index(key)
|
|
400
|
+
|
|
401
|
+
# Type 4: slice - Multiple items
|
|
402
|
+
elif isinstance(key, slice):
|
|
403
|
+
return self._get_by_slice(key)
|
|
404
|
+
|
|
405
|
+
# Type 5: callable - Filter function
|
|
406
|
+
elif callable(key):
|
|
407
|
+
return self._filter_by_function(key)
|
|
408
|
+
|
|
409
|
+
else:
|
|
410
|
+
raise TypeError(
|
|
411
|
+
f"Invalid key type: {type(key)}. Expected UUID, Progression, int, slice, or callable"
|
|
412
|
+
)
|
|
413
|
+
|
|
414
|
+
@synchronized
|
|
415
|
+
def _filter_by_progression(self, prog: Progression) -> Pile[T]:
|
|
416
|
+
"""Filter pile by progression order, returns new Pile."""
|
|
417
|
+
filtered_items = []
|
|
418
|
+
for uid in prog.order:
|
|
419
|
+
if uid in self._items:
|
|
420
|
+
filtered_items.append(self._items[uid])
|
|
421
|
+
|
|
422
|
+
# Return NEW Pile with filtered items
|
|
423
|
+
new_pile = Pile(
|
|
424
|
+
items=filtered_items,
|
|
425
|
+
item_type=self.item_type,
|
|
426
|
+
strict_type=self.strict_type,
|
|
427
|
+
)
|
|
428
|
+
return new_pile
|
|
429
|
+
|
|
430
|
+
@synchronized
|
|
431
|
+
def _get_by_index(self, index: int) -> T:
|
|
432
|
+
"""Get item by index in progression order."""
|
|
433
|
+
# With overloaded __getitem__, mypy knows int index returns UUID
|
|
434
|
+
uid: UUID = self._progression[index]
|
|
435
|
+
return self._items[uid]
|
|
436
|
+
|
|
437
|
+
@synchronized
|
|
438
|
+
def _get_by_slice(self, s: slice) -> list[T]:
|
|
439
|
+
"""Get multiple items by slice."""
|
|
440
|
+
# With overloaded __getitem__, mypy knows slice returns list[UUID]
|
|
441
|
+
uids: list[UUID] = self._progression[s]
|
|
442
|
+
return [self._items[uid] for uid in uids]
|
|
443
|
+
|
|
444
|
+
@synchronized
|
|
445
|
+
def _filter_by_function(self, func: Callable[[T], bool]) -> Pile[T]:
|
|
446
|
+
"""Filter pile by function - returns NEW Pile."""
|
|
447
|
+
filtered_items = [item for item in self if func(item)]
|
|
448
|
+
|
|
449
|
+
# Return NEW Pile with filtered items
|
|
450
|
+
new_pile = Pile(
|
|
451
|
+
items=filtered_items,
|
|
452
|
+
item_type=self.item_type,
|
|
453
|
+
strict_type=self.strict_type,
|
|
454
|
+
)
|
|
455
|
+
return new_pile
|
|
456
|
+
|
|
457
|
+
@synchronized
|
|
458
|
+
def filter_by_type(self, item_type: type[T] | set[type] | list[type]) -> Pile[T]:
|
|
459
|
+
"""Filter by type(s), returns new Pile.
|
|
460
|
+
|
|
461
|
+
Args:
|
|
462
|
+
item_type: Type(s) to filter by
|
|
463
|
+
|
|
464
|
+
Returns:
|
|
465
|
+
New Pile with filtered items
|
|
466
|
+
|
|
467
|
+
Raises:
|
|
468
|
+
TypeError: If requested type is not allowed
|
|
469
|
+
ValueError: If no items of requested type exist
|
|
470
|
+
"""
|
|
471
|
+
# Normalize to set
|
|
472
|
+
types_to_filter = extract_types(item_type)
|
|
473
|
+
|
|
474
|
+
# Check if types are allowed
|
|
475
|
+
if self.item_type is not None:
|
|
476
|
+
if self.strict_type:
|
|
477
|
+
# Strict mode: exact type match (efficient set operation)
|
|
478
|
+
invalid_types = types_to_filter - self.item_type
|
|
479
|
+
if invalid_types:
|
|
480
|
+
raise TypeError(
|
|
481
|
+
f"Types {invalid_types} not allowed in pile (allowed: {self.item_type})"
|
|
482
|
+
)
|
|
483
|
+
else:
|
|
484
|
+
# Permissive mode: check subclass relationships
|
|
485
|
+
# Build set of all compatible types (requested types + their subclasses/superclasses in allowed set)
|
|
486
|
+
for t in types_to_filter:
|
|
487
|
+
is_compatible = any(
|
|
488
|
+
issubclass(t, allowed) or issubclass(allowed, t)
|
|
489
|
+
for allowed in self.item_type
|
|
490
|
+
)
|
|
491
|
+
if not is_compatible:
|
|
492
|
+
raise TypeError(
|
|
493
|
+
f"Type {t} not compatible with allowed types {self.item_type}"
|
|
494
|
+
)
|
|
495
|
+
|
|
496
|
+
# Filter items by type(s)
|
|
497
|
+
filtered_items = [
|
|
498
|
+
item
|
|
499
|
+
for item in self._items.values()
|
|
500
|
+
if any(isinstance(item, t) for t in types_to_filter)
|
|
501
|
+
]
|
|
502
|
+
|
|
503
|
+
# Check if any items found
|
|
504
|
+
if not filtered_items:
|
|
505
|
+
raise ValueError(f"No items of type(s) {types_to_filter} found in pile")
|
|
506
|
+
|
|
507
|
+
# Return NEW Pile with filtered items
|
|
508
|
+
new_pile = Pile(
|
|
509
|
+
items=filtered_items,
|
|
510
|
+
item_type=self.item_type,
|
|
511
|
+
strict_type=self.strict_type,
|
|
512
|
+
)
|
|
513
|
+
return new_pile
|
|
514
|
+
|
|
515
|
+
# ==================== Context Managers ====================
|
|
516
|
+
|
|
517
|
+
async def __aenter__(self) -> Pile[T]:
|
|
518
|
+
"""Acquire lock for async context manager."""
|
|
519
|
+
await self._async_lock.acquire()
|
|
520
|
+
return self
|
|
521
|
+
|
|
522
|
+
async def __aexit__(self, _exc_type, _exc_val, _exc_tb) -> None:
|
|
523
|
+
"""Release lock for async context manager."""
|
|
524
|
+
self._async_lock.release()
|
|
525
|
+
|
|
526
|
+
# ==================== Async Operations ====================
|
|
527
|
+
|
|
528
|
+
@async_synchronized
|
|
529
|
+
async def add_async(self, item: T) -> None:
|
|
530
|
+
"""Async version of add()."""
|
|
531
|
+
self._validate_type(item)
|
|
532
|
+
|
|
533
|
+
if item.id in self._items:
|
|
534
|
+
raise ValueError(f"Item {item.id} already exists in pile")
|
|
535
|
+
|
|
536
|
+
self._items[item.id] = item
|
|
537
|
+
self._progression.append(item.id)
|
|
538
|
+
|
|
539
|
+
@async_synchronized
|
|
540
|
+
async def remove_async(self, item_id: UUID | str | Element) -> T:
|
|
541
|
+
"""Async version of remove()."""
|
|
542
|
+
|
|
543
|
+
uid = to_uuid(item_id)
|
|
544
|
+
|
|
545
|
+
if uid not in self._items:
|
|
546
|
+
raise ValueError(f"Item {uid} not found in pile")
|
|
547
|
+
|
|
548
|
+
item = self._items.pop(uid)
|
|
549
|
+
self._progression.remove(uid)
|
|
550
|
+
return item
|
|
551
|
+
|
|
552
|
+
@async_synchronized
|
|
553
|
+
async def get_async(self, item_id: UUID | str | Element) -> T:
|
|
554
|
+
"""Async version of get()."""
|
|
555
|
+
|
|
556
|
+
uid = to_uuid(item_id)
|
|
557
|
+
|
|
558
|
+
if uid not in self._items:
|
|
559
|
+
raise ValueError(f"Item {uid} not found in pile")
|
|
560
|
+
|
|
561
|
+
return self._items[uid]
|
|
562
|
+
|
|
563
|
+
# ==================== Query Operations ====================
|
|
564
|
+
|
|
565
|
+
@synchronized
|
|
566
|
+
def __contains__(self, item: UUID | str | Element) -> bool:
|
|
567
|
+
"""Check if item exists in pile."""
|
|
568
|
+
with contextlib.suppress(Exception):
|
|
569
|
+
uid = to_uuid(item)
|
|
570
|
+
return uid in self._items
|
|
571
|
+
return False
|
|
572
|
+
|
|
573
|
+
@synchronized
|
|
574
|
+
def __len__(self) -> int:
|
|
575
|
+
"""Return number of items."""
|
|
576
|
+
return len(self._items)
|
|
577
|
+
|
|
578
|
+
@synchronized
|
|
579
|
+
def __iter__(self) -> Iterator[T]:
|
|
580
|
+
"""Iterate items in insertion order."""
|
|
581
|
+
for uid in self._progression:
|
|
582
|
+
if uid in self._items:
|
|
583
|
+
yield self._items[uid]
|
|
584
|
+
|
|
585
|
+
def __list__(self) -> list[T]:
|
|
586
|
+
"""Return items as list in insertion order."""
|
|
587
|
+
return [self._items[uid] for uid in self._progression if uid in self._items]
|
|
588
|
+
|
|
589
|
+
@synchronized
|
|
590
|
+
def to_list(self) -> list[T]:
|
|
591
|
+
"""Return items as list in insertion order."""
|
|
592
|
+
return list(self)
|
|
593
|
+
|
|
594
|
+
@synchronized
|
|
595
|
+
def keys(self):
|
|
596
|
+
"""Yield UUIDs of items."""
|
|
597
|
+
yield from self._items.keys()
|
|
598
|
+
|
|
599
|
+
@synchronized
|
|
600
|
+
def values(self):
|
|
601
|
+
"""Yield items in insertion order."""
|
|
602
|
+
yield from self
|
|
603
|
+
|
|
604
|
+
def size(self) -> int:
|
|
605
|
+
"""Return number of items (alias for __len__)."""
|
|
606
|
+
return len(self)
|
|
607
|
+
|
|
608
|
+
def is_empty(self) -> bool:
|
|
609
|
+
"""Check if pile is empty."""
|
|
610
|
+
return len(self._items) == 0
|
|
611
|
+
|
|
612
|
+
# ==================== Validation ====================
|
|
613
|
+
|
|
614
|
+
def _validate_type(self, item: T) -> None:
|
|
615
|
+
"""Validate item type with set-based checking and strict mode."""
|
|
616
|
+
if not isinstance(item, Element):
|
|
617
|
+
raise TypeError(f"Item must be Element subclass, got {type(item)}")
|
|
618
|
+
|
|
619
|
+
if self.item_type is not None:
|
|
620
|
+
item_type_actual = type(item)
|
|
621
|
+
|
|
622
|
+
if self.strict_type:
|
|
623
|
+
# Strict mode: exact type match only (no subclasses)
|
|
624
|
+
if item_type_actual not in self.item_type:
|
|
625
|
+
raise TypeError(
|
|
626
|
+
f"Item type {item_type_actual} not in allowed types {self.item_type} "
|
|
627
|
+
"(strict_type=True, no subclasses allowed)"
|
|
628
|
+
)
|
|
629
|
+
else:
|
|
630
|
+
# Permissive mode: allow subclasses
|
|
631
|
+
if not any(issubclass(item_type_actual, t) for t in self.item_type):
|
|
632
|
+
raise TypeError(
|
|
633
|
+
f"Item type {item_type_actual} is not a subclass of any allowed type {self.item_type}"
|
|
634
|
+
)
|
|
635
|
+
|
|
636
|
+
# ==================== Deserialization ====================
|
|
637
|
+
|
|
638
|
+
@classmethod
|
|
639
|
+
@override
|
|
640
|
+
def from_dict(
|
|
641
|
+
cls,
|
|
642
|
+
data: dict[str, Any],
|
|
643
|
+
meta_key: str | None = None,
|
|
644
|
+
item_meta_key: str | None = None,
|
|
645
|
+
**kwargs: Any,
|
|
646
|
+
) -> Pile[T]:
|
|
647
|
+
"""Deserialize Pile from dict.
|
|
648
|
+
|
|
649
|
+
Args:
|
|
650
|
+
data: Serialized pile data
|
|
651
|
+
meta_key: If provided, rename this key back to "metadata" (for db mode deserialization)
|
|
652
|
+
item_meta_key: If provided, pass to Element.from_dict for item deserialization
|
|
653
|
+
**kwargs: Additional arguments, including optional item_type
|
|
654
|
+
|
|
655
|
+
Returns:
|
|
656
|
+
Reconstructed Pile
|
|
657
|
+
"""
|
|
658
|
+
from .element import Element
|
|
659
|
+
|
|
660
|
+
# Make a copy to avoid mutating input
|
|
661
|
+
data = data.copy()
|
|
662
|
+
|
|
663
|
+
# Restore metadata from custom key if specified (db mode deserialization)
|
|
664
|
+
if meta_key and meta_key in data:
|
|
665
|
+
data["metadata"] = data.pop(meta_key)
|
|
666
|
+
|
|
667
|
+
# Extract pile configuration
|
|
668
|
+
item_type_data = data.get("item_type") or kwargs.get("item_type")
|
|
669
|
+
strict_type = data.get("strict_type", False)
|
|
670
|
+
|
|
671
|
+
# FAIL FAST: Validate ALL item types before deserializing ANY items
|
|
672
|
+
items_data = data.get("items", [])
|
|
673
|
+
if item_type_data is not None and items_data:
|
|
674
|
+
# Normalize item_type to set of types (handle serialized strings)
|
|
675
|
+
if (
|
|
676
|
+
isinstance(item_type_data, list)
|
|
677
|
+
and item_type_data
|
|
678
|
+
and isinstance(item_type_data[0], str)
|
|
679
|
+
):
|
|
680
|
+
# Deserialization case: convert strings to types
|
|
681
|
+
allowed_types = {load_type_from_string(type_str) for type_str in item_type_data}
|
|
682
|
+
else:
|
|
683
|
+
# Runtime case: use extract_types
|
|
684
|
+
allowed_types = extract_types(item_type_data)
|
|
685
|
+
|
|
686
|
+
# Validate all lion_class values upfront
|
|
687
|
+
for item_dict in items_data:
|
|
688
|
+
lion_class = item_dict.get("metadata", {}).get("lion_class")
|
|
689
|
+
if lion_class:
|
|
690
|
+
try:
|
|
691
|
+
item_type_actual = load_type_from_string(lion_class)
|
|
692
|
+
except ValueError:
|
|
693
|
+
# Let Element.from_dict handle invalid types
|
|
694
|
+
continue
|
|
695
|
+
|
|
696
|
+
# Check type compatibility
|
|
697
|
+
if strict_type:
|
|
698
|
+
# Strict: exact type match
|
|
699
|
+
if item_type_actual not in allowed_types:
|
|
700
|
+
raise TypeError(
|
|
701
|
+
f"Item type {lion_class} not in allowed types {allowed_types} "
|
|
702
|
+
"(strict_type=True)"
|
|
703
|
+
)
|
|
704
|
+
else:
|
|
705
|
+
# Permissive: allow subclasses
|
|
706
|
+
if not any(issubclass(item_type_actual, t) for t in allowed_types):
|
|
707
|
+
raise TypeError(
|
|
708
|
+
f"Item type {lion_class} is not a subclass of any allowed type {allowed_types}"
|
|
709
|
+
)
|
|
710
|
+
|
|
711
|
+
# Create pile with Element fields (id, created_at, metadata preserved)
|
|
712
|
+
# Remove items/progression/item_type/strict_type from data to avoid duplication
|
|
713
|
+
pile_data = data.copy()
|
|
714
|
+
pile_data.pop("items", None)
|
|
715
|
+
pile_data.pop("item_type", None)
|
|
716
|
+
pile_data.pop("strict_type", None)
|
|
717
|
+
pile = cls(item_type=item_type_data, strict_type=strict_type, **pile_data)
|
|
718
|
+
|
|
719
|
+
# Extract and restore progression metadata
|
|
720
|
+
metadata = data.get("metadata", {})
|
|
721
|
+
progression_name = metadata.get("progression_name")
|
|
722
|
+
if progression_name:
|
|
723
|
+
pile._progression.name = progression_name
|
|
724
|
+
|
|
725
|
+
# Deserialize items (type validation already done above)
|
|
726
|
+
for item_dict in items_data:
|
|
727
|
+
item = Element.from_dict(item_dict, meta_key=item_meta_key)
|
|
728
|
+
pile.add(item) # Adds to _items dict + _progression (maintains order)
|
|
729
|
+
|
|
730
|
+
return pile
|
|
731
|
+
|
|
732
|
+
# ==================== Adapter Methods ====================
|
|
733
|
+
|
|
734
|
+
def adapt_to(self, obj_key: str, many: bool = False, **kwargs: Any) -> Any:
|
|
735
|
+
"""Convert to external format via pydapter adapter.
|
|
736
|
+
|
|
737
|
+
Args:
|
|
738
|
+
obj_key: Adapter key (e.g., "postgres", "qdrant", "mongo")
|
|
739
|
+
many: Adaptation mode:
|
|
740
|
+
- False (default): Treat entire Pile as single object (export Pile + all items)
|
|
741
|
+
- True: Bulk operation on items within Pile (export items individually)
|
|
742
|
+
**kwargs: Passed to adapter
|
|
743
|
+
|
|
744
|
+
Returns:
|
|
745
|
+
Adapted object (format depends on adapter and many mode)
|
|
746
|
+
|
|
747
|
+
Note:
|
|
748
|
+
For many=True, adapter operates on self.items (bulk insert/export items).
|
|
749
|
+
For many=False, adapter operates on entire Pile (single object serialization).
|
|
750
|
+
"""
|
|
751
|
+
kwargs.setdefault("adapt_meth", "to_dict")
|
|
752
|
+
kwargs.setdefault("adapt_kw", {"mode": "db"})
|
|
753
|
+
return super().adapt_to(obj_key=obj_key, many=many, **kwargs)
|
|
754
|
+
|
|
755
|
+
@classmethod
|
|
756
|
+
def adapt_from(cls, obj: Any, obj_key: str, many: bool = False, **kwargs: Any) -> Pile:
|
|
757
|
+
"""Create from external format via pydapter adapter.
|
|
758
|
+
|
|
759
|
+
Args:
|
|
760
|
+
obj: Source object
|
|
761
|
+
obj_key: Adapter key (e.g., "postgres", "qdrant", "mongo")
|
|
762
|
+
many: Adaptation mode:
|
|
763
|
+
- False (default): Deserialize entire Pile from single object
|
|
764
|
+
- True: Bulk load items from external source (construct Pile from items)
|
|
765
|
+
**kwargs: Passed to adapter
|
|
766
|
+
|
|
767
|
+
Returns:
|
|
768
|
+
Pile instance
|
|
769
|
+
|
|
770
|
+
Note:
|
|
771
|
+
For many=True, adapter fetches multiple items and constructs Pile.
|
|
772
|
+
For many=False, adapter deserializes pre-serialized Pile object.
|
|
773
|
+
"""
|
|
774
|
+
kwargs.setdefault("adapt_meth", "from_dict")
|
|
775
|
+
return super().adapt_from(obj, obj_key=obj_key, many=many, **kwargs)
|
|
776
|
+
|
|
777
|
+
async def adapt_to_async(self, obj_key: str, many: bool = False, **kwargs: Any) -> Any:
|
|
778
|
+
"""Async convert to external format via pydapter async adapter.
|
|
779
|
+
|
|
780
|
+
Args:
|
|
781
|
+
obj_key: Adapter key
|
|
782
|
+
many: Adaptation mode (see adapt_to for details)
|
|
783
|
+
**kwargs: Passed to adapter
|
|
784
|
+
|
|
785
|
+
Returns:
|
|
786
|
+
Adapted object
|
|
787
|
+
"""
|
|
788
|
+
kwargs.setdefault("adapt_meth", "to_dict")
|
|
789
|
+
kwargs.setdefault("adapt_kw", {"mode": "db"})
|
|
790
|
+
return await super().adapt_to_async(obj_key=obj_key, many=many, **kwargs)
|
|
791
|
+
|
|
792
|
+
@classmethod
|
|
793
|
+
async def adapt_from_async(
|
|
794
|
+
cls, obj: Any, obj_key: str, many: bool = False, **kwargs: Any
|
|
795
|
+
) -> Pile:
|
|
796
|
+
"""Async create from external format via pydapter async adapter.
|
|
797
|
+
|
|
798
|
+
Args:
|
|
799
|
+
obj: Source object
|
|
800
|
+
obj_key: Adapter key
|
|
801
|
+
many: Adaptation mode (see adapt_from for details)
|
|
802
|
+
**kwargs: Passed to adapter
|
|
803
|
+
|
|
804
|
+
Returns:
|
|
805
|
+
Pile instance
|
|
806
|
+
"""
|
|
807
|
+
kwargs.setdefault("adapt_meth", "from_dict")
|
|
808
|
+
return await super().adapt_from_async(obj, obj_key=obj_key, many=many, **kwargs)
|
|
809
|
+
|
|
810
|
+
def __repr__(self) -> str:
|
|
811
|
+
return f"Pile(len={len(self)})"
|