brainlessdb 0.17.4__tar.gz → 1.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. brainlessdb-1.1.0/.python-version +1 -0
  2. {brainlessdb-0.17.4 → brainlessdb-1.1.0}/CHANGELOG.md +35 -0
  3. {brainlessdb-0.17.4 → brainlessdb-1.1.0}/PKG-INFO +30 -1
  4. {brainlessdb-0.17.4 → brainlessdb-1.1.0}/README.md +29 -0
  5. {brainlessdb-0.17.4 → brainlessdb-1.1.0}/brainlessdb/__init__.py +4 -1
  6. {brainlessdb-0.17.4 → brainlessdb-1.1.0}/brainlessdb/collection.py +22 -0
  7. brainlessdb-1.1.0/brainlessdb/ingest.py +225 -0
  8. {brainlessdb-0.17.4 → brainlessdb-1.1.0}/brainlessdb/struct.py +43 -8
  9. {brainlessdb-0.17.4 → brainlessdb-1.1.0}/pyproject.toml +1 -1
  10. brainlessdb-1.1.0/tests/test_ingest.py +193 -0
  11. {brainlessdb-0.17.4 → brainlessdb-1.1.0}/uv.lock +1 -1
  12. brainlessdb-0.17.4/.python-version +0 -1
  13. {brainlessdb-0.17.4 → brainlessdb-1.1.0}/.gitignore +0 -0
  14. {brainlessdb-0.17.4 → brainlessdb-1.1.0}/brainlessdb/bucket.py +0 -0
  15. {brainlessdb-0.17.4 → brainlessdb-1.1.0}/brainlessdb/client.py +0 -0
  16. {brainlessdb-0.17.4 → brainlessdb-1.1.0}/brainlessdb/fields.py +0 -0
  17. {brainlessdb-0.17.4 → brainlessdb-1.1.0}/brainlessdb/registry.py +0 -0
  18. {brainlessdb-0.17.4 → brainlessdb-1.1.0}/cliff.toml +0 -0
  19. {brainlessdb-0.17.4 → brainlessdb-1.1.0}/code_style_guide.md +0 -0
  20. {brainlessdb-0.17.4 → brainlessdb-1.1.0}/justfile +0 -0
  21. {brainlessdb-0.17.4 → brainlessdb-1.1.0}/test_results.txt +0 -0
  22. {brainlessdb-0.17.4 → brainlessdb-1.1.0}/tests/__init__.py +0 -0
  23. {brainlessdb-0.17.4 → brainlessdb-1.1.0}/tests/conftest.py +0 -0
  24. {brainlessdb-0.17.4 → brainlessdb-1.1.0}/tests/mock_nats.py +0 -0
  25. {brainlessdb-0.17.4 → brainlessdb-1.1.0}/tests/test_bucket.py +0 -0
  26. {brainlessdb-0.17.4 → brainlessdb-1.1.0}/tests/test_client.py +0 -0
  27. {brainlessdb-0.17.4 → brainlessdb-1.1.0}/tests/test_collection.py +0 -0
  28. {brainlessdb-0.17.4 → brainlessdb-1.1.0}/tests/test_dead_letter.py +0 -0
  29. {brainlessdb-0.17.4 → brainlessdb-1.1.0}/tests/test_deserialization.py +0 -0
  30. {brainlessdb-0.17.4 → brainlessdb-1.1.0}/tests/test_duplicates.py +0 -0
  31. {brainlessdb-0.17.4 → brainlessdb-1.1.0}/tests/test_fields.py +0 -0
  32. {brainlessdb-0.17.4 → brainlessdb-1.1.0}/tests/test_flush_errors.py +0 -0
  33. {brainlessdb-0.17.4 → brainlessdb-1.1.0}/tests/test_load_errors.py +0 -0
  34. {brainlessdb-0.17.4 → brainlessdb-1.1.0}/tests/test_registry.py +0 -0
  35. {brainlessdb-0.17.4 → brainlessdb-1.1.0}/tests/test_sync.py +0 -0
  36. {brainlessdb-0.17.4 → brainlessdb-1.1.0}/tests/test_thread_safety.py +0 -0
  37. {brainlessdb-0.17.4 → brainlessdb-1.1.0}/tests/test_update_atomic.py +0 -0
  38. {brainlessdb-0.17.4 → brainlessdb-1.1.0}/tests/test_version.py +0 -0
@@ -0,0 +1 @@
1
+ 3.11
@@ -3,6 +3,41 @@
3
3
  All notable changes to this project will be documented in this file.
4
4
 
5
5
 
6
+ ## [1.1.0] - 2026-11-03
7
+
8
+ ### 🚀 Features
9
+
10
+ - Ingest() reports unknown keys
11
+ - Top level keys which are not fields of the struct were silently left out,
12
+ so a typo or a renamed field went unnoticed. They are still left out, but
13
+ now reported as Repair with action "ignored".
14
+
15
+ ### ⚙️ Miscellaneous Tasks
16
+
17
+ - Update uv.lock to 1.0.0
18
+
19
+
20
+ ## [1.0.0] - 2026-11-03
21
+
22
+ ### 🚀 Features
23
+
24
+ - Ingest() to store external data with invalid values repaired
25
+ - add(), update() and save() raise on any invalid value, so one bad column
26
+ from SQL dropped the whole item. Collection.ingest(data, key),
27
+ BrainlessBucket.ingest(data) and BrainlessRegistry.ingest(data) repair
28
+ instead: an invalid value keeps the current one, a new item gets the field
29
+ default, invalid list items and dict entries are removed. Values are
30
+ coerced like msgspec.convert(strict=False).
31
+
32
+ Repairs are returned (path, action, error) without the invalid value,
33
+ which may be a secret. Struct types are walked directly, so msgspec error
34
+ messages are not parsed. List items are not matched to current ones yet.
35
+
36
+ ### ⚙️ Miscellaneous Tasks
37
+
38
+ - [**breaking**] Change minimal required Python version from 3.9 to 3.11
39
+
40
+
6
41
  ## [0.17.4] - 2026-11-03
7
42
 
8
43
  ### 🐛 Bug Fixes
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: brainlessdb
3
- Version: 0.17.4
3
+ Version: 1.1.0
4
4
  Summary: Typed collections backed by NATS JetStream KV
5
5
  Author-email: "INSOFT s.r.o." <helpdesk@insoft.cz>
6
6
  License-Expression: MIT
@@ -205,6 +205,35 @@ for item in coll: ...
205
205
  item in coll
206
206
  ```
207
207
 
208
+ ## Ingest
209
+
210
+ For data from outside (SQL rows, config files) that may not match the struct.
211
+ Where `add()`, `update()` and `save()` raise on an invalid value, `ingest()`
212
+ repairs it and reports what it did:
213
+
214
+ ```python
215
+ # add or update item found by a UNIQUE field
216
+ item, repairs = coll.ingest(row, key="id")
217
+
218
+ # partial update of an item, like update()
219
+ repairs = item.ingest({"vlan": row["vlan"]})
220
+
221
+ # registry
222
+ repairs = cfg.ingest({"port": "5070"}) # coerced to 5070
223
+
224
+ for r in repairs:
225
+ log.warning("%s %s: %s", ".".join(map(str, r.path)), r.action, r.error)
226
+ ```
227
+
228
+ An invalid value keeps the current one (`kept`), or uses the field default on
229
+ a new item (`default`). Invalid list items and dict entries are removed
230
+ (`removed`); list items are not matched to current ones by position. Values
231
+ are coerced like `msgspec.convert(strict=False)`, e.g. `"5070"` to `5070`.
232
+ A repair records the path and error, never the invalid value, which may be a
233
+ password. Data that can't be repaired (an invalid required field of a new
234
+ item) raises `msgspec.ValidationError`, unique violations raise
235
+ `UniqueConstraintError` as usual.
236
+
208
237
  ## Filtering
209
238
 
210
239
  All filter/find methods are **sync** (operate on in-memory data):
@@ -194,6 +194,35 @@ for item in coll: ...
194
194
  item in coll
195
195
  ```
196
196
 
197
+ ## Ingest
198
+
199
+ For data from outside (SQL rows, config files) that may not match the struct.
200
+ Where `add()`, `update()` and `save()` raise on an invalid value, `ingest()`
201
+ repairs it and reports what it did:
202
+
203
+ ```python
204
+ # add or update item found by a UNIQUE field
205
+ item, repairs = coll.ingest(row, key="id")
206
+
207
+ # partial update of an item, like update()
208
+ repairs = item.ingest({"vlan": row["vlan"]})
209
+
210
+ # registry
211
+ repairs = cfg.ingest({"port": "5070"}) # coerced to 5070
212
+
213
+ for r in repairs:
214
+ log.warning("%s %s: %s", ".".join(map(str, r.path)), r.action, r.error)
215
+ ```
216
+
217
+ An invalid value keeps the current one (`kept`), or uses the field default on
218
+ a new item (`default`). Invalid list items and dict entries are removed
219
+ (`removed`); list items are not matched to current ones by position. Values
220
+ are coerced like `msgspec.convert(strict=False)`, e.g. `"5070"` to `5070`.
221
+ A repair records the path and error, never the invalid value, which may be a
222
+ password. Data that can't be repaired (an invalid required field of a new
223
+ item) raises `msgspec.ValidationError`, unique violations raise
224
+ `UniqueConstraintError` as usual.
225
+
197
226
  ## Filtering
198
227
 
199
228
  All filter/find methods are **sync** (operate on in-memory data):
@@ -1,12 +1,13 @@
1
1
  """Brainless DB - Typed collections backed by NATS JetStream KV."""
2
2
 
3
- __version__ = "0.17.4"
3
+ __version__ = "1.1.0"
4
4
 
5
5
  from typing import Any, Optional, TypeVar
6
6
 
7
7
  from .bucket import ShortReadError
8
8
  from .client import BrainlessDB
9
9
  from .collection import Collection, HasUUID
10
+ from .ingest import IngestResult, Repair
10
11
  from .registry import Registry
11
12
  from .struct import (
12
13
  AppAccessError,
@@ -27,6 +28,8 @@ __all__ = [
27
28
  "Collection",
28
29
  "Registry",
29
30
  "HasUUID",
31
+ "IngestResult",
32
+ "Repair",
30
33
  "AppAccessError",
31
34
  "ShortReadError",
32
35
  "UniqueConstraintError",
@@ -12,6 +12,7 @@ import msgspec
12
12
 
13
13
  from .bucket import Bucket
14
14
  from .fields import analyze_fields
15
+ from .ingest import IngestResult, repair
15
16
  from .struct import (
16
17
  BrainlessBucket,
17
18
  ConfigWrapper,
@@ -512,6 +513,27 @@ class Collection(Generic[T]):
512
513
  self._fire_change(old, obj, local=True)
513
514
  return obj
514
515
 
516
+ def ingest(self, data: dict[str, Any], key: str) -> IngestResult:
517
+ """Add or update item found by unique field `key`, repairing invalid values.
518
+
519
+ Update works like item.ingest(): an invalid value keeps the current one.
520
+ A new item gets the field default instead. Inside lists and dicts the
521
+ invalid item is removed. Raises msgspec.ValidationError when data can't be
522
+ repaired, e.g. a new item with invalid required field.
523
+ """
524
+ if key not in self._analysis.unique_fields:
525
+ raise ValueError(f"{self._cls.__name__}.{key} is not a unique field")
526
+ if key not in data:
527
+ raise ValueError(f"Missing key field {key!r} in data")
528
+
529
+ item = self.find(**{key: data[key]})
530
+ if item is not None:
531
+ repairs = item.ingest(data)
532
+ return IngestResult(self.get(str(item.uuid)), repairs)
533
+
534
+ fixed, repairs = repair(data, self._cls)
535
+ return IngestResult(self.add(fixed), repairs)
536
+
515
537
  def get(self, uuid: str) -> Optional[T]:
516
538
  return self._entities.get(uuid)
517
539
 
@@ -0,0 +1,225 @@
1
+ """Repair of external data so it fits a struct, for ingest()."""
2
+
3
+ from typing import Annotated, Any, NamedTuple, Optional, Union, get_args, get_origin
4
+
5
+ import msgspec
6
+ from msgspec import Struct
7
+
8
+ Path = tuple[Union[str, int], ...]
9
+
10
+ _MISSING = object()
11
+
12
+
13
+ class Repair(Struct, frozen=True):
14
+ """One value ingest() could not store as given.
15
+
16
+ action is "kept" (current value kept), "default" (field default used),
17
+ "removed" (list item or dict entry left out) or "ignored" (top level key
18
+ is not a field of the struct). The invalid value itself is not recorded,
19
+ it may be a password or key.
20
+ """
21
+
22
+ path: Path
23
+ action: str
24
+ error: str
25
+
26
+
27
+ class IngestResult(NamedTuple):
28
+ """Stored item and repairs made to store it."""
29
+
30
+ item: Any
31
+ repairs: list[Repair]
32
+
33
+
34
+ class _Unrepairable(Exception):
35
+ """Value can't be repaired at its level, the level above has to drop it."""
36
+
37
+ def __init__(self, error: str) -> None:
38
+ super().__init__(error)
39
+ self.error = error
40
+
41
+
42
+ def _check(value: Any, tp: Any) -> Optional[str]:
43
+ """Validation error message, None when value is valid."""
44
+ try:
45
+ msgspec.convert(value, tp, strict=False)
46
+ except msgspec.ValidationError as ex:
47
+ return str(ex)
48
+ return None
49
+
50
+
51
+ def _unwrap(tp: Any) -> list[Any]:
52
+ """Candidate types of a hint: Annotated removed, Union split, None left out."""
53
+ if get_origin(tp) is Annotated:
54
+ tp = get_args(tp)[0]
55
+ if get_origin(tp) is Union or type(tp).__name__ == "UnionType":
56
+ result = []
57
+ for arg in get_args(tp):
58
+ result.extend(_unwrap(arg))
59
+ return result
60
+ return [] if tp is type(None) else [tp]
61
+
62
+
63
+ def _struct_for(candidates: list[Any], value: dict) -> Optional[type]:
64
+ """Struct type the dict value belongs to, using the tag for tagged unions."""
65
+ structs = [c for c in candidates if isinstance(c, type) and issubclass(c, Struct)]
66
+ for cls in structs:
67
+ config = cls.__struct_config__
68
+ if config.tag is not None and value.get(config.tag_field) == config.tag:
69
+ return cls
70
+ return structs[0] if len(structs) == 1 else None
71
+
72
+
73
+ def _repair_value(value: Any, tp: Any, current: Any, path: Path, repairs: list[Repair]) -> Any:
74
+ """Repaired value, raises _Unrepairable when the value has to be dropped."""
75
+ error = _check(value, tp)
76
+ if error is None:
77
+ return value
78
+
79
+ local: list[Repair] = []
80
+ candidates = _unwrap(tp)
81
+ fixed: Any = _MISSING
82
+
83
+ if isinstance(value, dict):
84
+ cls = _struct_for(candidates, value)
85
+ if cls is not None:
86
+ fixed = _repair_struct(value, cls, current if isinstance(current, dict) else None, path, local)
87
+ else:
88
+ dict_type = next((c for c in candidates if get_origin(c) is dict), None)
89
+ if dict_type is not None:
90
+ fixed = _repair_dict(value, dict_type, path, local)
91
+ elif isinstance(value, list):
92
+ list_type = next((c for c in candidates if _item_type(c) is not None), None)
93
+ if list_type is not None:
94
+ fixed = _repair_list(value, _item_type(list_type), path, local)
95
+
96
+ if fixed is _MISSING:
97
+ raise _Unrepairable(error)
98
+
99
+ # constraints of the container itself, e.g. min_length, may still fail
100
+ error = _check(fixed, tp)
101
+ if error is not None:
102
+ raise _Unrepairable(error)
103
+
104
+ repairs.extend(local)
105
+ return fixed
106
+
107
+
108
+ def _item_type(tp: Any) -> Any:
109
+ """Item type of list-like type, None for other types."""
110
+ origin = get_origin(tp)
111
+ args = get_args(tp)
112
+ if origin in (list, set, frozenset) and args:
113
+ return args[0]
114
+ if origin is tuple and len(args) == 2 and args[1] is Ellipsis:
115
+ return args[0]
116
+ return None
117
+
118
+
119
+ def _repair_list(value: list, item_type: Any, path: Path, repairs: list[Repair]) -> list:
120
+ """Invalid items are removed. Items are not matched to current ones, positions may differ."""
121
+ fixed = []
122
+ for index, item in enumerate(value):
123
+ try:
124
+ fixed.append(_repair_value(item, item_type, None, (*path, index), repairs))
125
+ except _Unrepairable as ex:
126
+ repairs.append(Repair((*path, index), "removed", ex.error))
127
+ return fixed
128
+
129
+
130
+ def _repair_dict(value: dict, dict_type: Any, path: Path, repairs: list[Repair]) -> dict:
131
+ """Entries with invalid key or value are removed."""
132
+ key_type, value_type = get_args(dict_type)
133
+ fixed = {}
134
+ for key, item in value.items():
135
+ error = _check(key, key_type)
136
+ if error is not None:
137
+ repairs.append(Repair((*path, key), "removed", error))
138
+ continue
139
+ try:
140
+ fixed[key] = _repair_value(item, value_type, None, (*path, key), repairs)
141
+ except _Unrepairable as ex:
142
+ repairs.append(Repair((*path, key), "removed", ex.error))
143
+ return fixed
144
+
145
+
146
+ def _repair_struct(
147
+ value: dict,
148
+ cls: type,
149
+ current: Optional[dict],
150
+ path: Path,
151
+ repairs: list[Repair],
152
+ *,
153
+ partial: bool = False,
154
+ ) -> dict:
155
+ """Repair fields of struct dict.
156
+
157
+ Invalid field keeps its current value when known, otherwise it is left out so
158
+ its default is used. Without either the whole struct is unrepairable. With
159
+ partial=True fields missing in value are not required (update semantics).
160
+ """
161
+ fixed = dict(value)
162
+ local: list[Repair] = []
163
+ for field in msgspec.structs.fields(cls):
164
+ key = field.encode_name
165
+ field_path = (*path, key)
166
+ known = current.get(key, _MISSING) if current is not None else _MISSING
167
+
168
+ if key not in fixed:
169
+ if partial or not field.required:
170
+ continue
171
+ if known is _MISSING:
172
+ raise _Unrepairable(f"Object missing required field `{key}`")
173
+ fixed[key] = known
174
+ local.append(Repair(field_path, "kept", f"Object missing required field `{key}`"))
175
+ continue
176
+
177
+ try:
178
+ fixed[key] = _repair_value(fixed[key], field.type, known, field_path, local)
179
+ except _Unrepairable as ex:
180
+ if known is not _MISSING:
181
+ fixed[key] = known
182
+ local.append(Repair(field_path, "kept", ex.error))
183
+ elif not field.required:
184
+ del fixed[key]
185
+ local.append(Repair(field_path, "default", ex.error))
186
+ else:
187
+ raise
188
+
189
+ repairs.extend(local)
190
+ return fixed
191
+
192
+
193
+ def repair(
194
+ data: dict[str, Any],
195
+ cls: type,
196
+ current: Optional[dict[str, Any]] = None,
197
+ *,
198
+ partial: bool = False,
199
+ ) -> tuple[dict[str, Any], list[Repair]]:
200
+ """Make data valid for cls, return it converted to builtins with the repairs made.
201
+
202
+ current is the stored item as builtins: invalid values fall back to it, except
203
+ inside lists and dicts. Values are coerced like msgspec.convert(strict=False),
204
+ e.g. "5060" to 5060. Raises msgspec.ValidationError when data can't be repaired,
205
+ e.g. a required field without current value is invalid. Top level keys
206
+ which are not fields of cls are left out and reported as "ignored".
207
+ """
208
+ fields = {field.encode_name: field for field in msgspec.structs.fields(cls)}
209
+ tag_field = cls.__struct_config__.tag_field
210
+ repairs = [
211
+ Repair((key,), "ignored", f"Object has unknown field `{key}`")
212
+ for key in data
213
+ if key not in fields and key != tag_field
214
+ ]
215
+ try:
216
+ fixed = _repair_struct(data, cls, current, (), repairs, partial=partial)
217
+ except _Unrepairable as ex:
218
+ raise msgspec.ValidationError(ex.error) from None
219
+
220
+ converted = {
221
+ key: msgspec.to_builtins(msgspec.convert(value, fields[key].type, strict=False))
222
+ for key, value in fixed.items()
223
+ if key in fields
224
+ }
225
+ return converted, repairs
@@ -10,6 +10,8 @@ from weakref import WeakValueDictionary
10
10
  import msgspec
11
11
  from msgspec import Struct, convert, field, to_builtins
12
12
 
13
+ from .ingest import Repair, repair
14
+
13
15
  _VERSION_RE = re.compile(r"^(.+?)V(\d+)$")
14
16
 
15
17
 
@@ -157,14 +159,7 @@ class BrainlessBucket(Struct, kw_only=True, tag=True, tag_field="TYPE"):
157
159
  stores its own validated copy, so re-fetch to get the stored object.
158
160
  """
159
161
  fields = self.__struct_fields__
160
- merged = to_builtins(self)
161
- for key, value in data.items():
162
- if key not in fields:
163
- continue
164
- current = getattr(self, key)
165
- if isinstance(value, dict) and hasattr(current, "__struct_fields__"):
166
- value = {**to_builtins(current), **value}
167
- merged[key] = to_builtins(value)
162
+ merged = {**to_builtins(self), **_merge_nested(self, data)}
168
163
  new = convert(merged, type(self))
169
164
  collection = _registry.get(self._uuid)
170
165
  if collection is not None:
@@ -173,6 +168,32 @@ class BrainlessBucket(Struct, kw_only=True, tag=True, tag_field="TYPE"):
173
168
  setattr(self, f, getattr(new, f))
174
169
  return self
175
170
 
171
+ def ingest(self, data: dict) -> list[Repair]:
172
+ """Like update(), but repair invalid values instead of raising.
173
+
174
+ An invalid value keeps the current one; inside lists and dicts the invalid
175
+ item is removed (items are not matched by position). Unknown keys are
176
+ ignored and reported. Values are coerced, e.g. "5060" to 5060. Returns
177
+ the repairs made. Raises like update() when data can't be repaired or
178
+ violates a unique constraint.
179
+ """
180
+ fixed, repairs = repair({**data, **_merge_nested(self, data)}, type(self), to_builtins(self), partial=True)
181
+ self.update(fixed)
182
+ return repairs
183
+
184
+
185
+ def _merge_nested(obj: Struct, data: dict) -> dict:
186
+ """Builtins of data keys which are fields of obj, dicts merged into nested structs."""
187
+ result = {}
188
+ for key, value in data.items():
189
+ if key not in obj.__struct_fields__:
190
+ continue
191
+ current = getattr(obj, key)
192
+ if isinstance(value, dict) and hasattr(current, "__struct_fields__"):
193
+ value = {**to_builtins(current), **value}
194
+ result[key] = to_builtins(value)
195
+ return result
196
+
176
197
 
177
198
  # Fixed namespace for deriving registry storage keys via uuid5.
178
199
  _REGISTRY_NS = uuid5(NAMESPACE_DNS, "brainlessdb.registry")
@@ -204,6 +225,20 @@ class BrainlessRegistry(Struct, kw_only=True, tag=True, tag_field="TYPE", dict=T
204
225
  def uuid(self) -> UUID:
205
226
  return uuid5(_REGISTRY_NS, type(self).__key__)
206
227
 
228
+ def ingest(self, data: dict) -> list[Repair]:
229
+ """Set fields from data and save, repairing invalid values instead of raising.
230
+
231
+ An invalid value keeps the current one, unknown keys are ignored and
232
+ reported. Values are coerced, e.g. "5060" to 5060. Returns the repairs made.
233
+ """
234
+ current = to_builtins(self)
235
+ fixed, repairs = repair({**data, **_merge_nested(self, data)}, type(self), current, partial=True)
236
+ new = convert({**current, **fixed}, type(self))
237
+ for f in self.__struct_fields__:
238
+ setattr(self, f, getattr(new, f))
239
+ self.save()
240
+ return repairs
241
+
207
242
  def save(self) -> None:
208
243
  """Validate and mark this registry entry dirty in its Registry.
209
244
 
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "brainlessdb"
3
- version = "0.17.4"
3
+ version = "1.1.0"
4
4
  description = "Typed collections backed by NATS JetStream KV"
5
5
  authors = [
6
6
  {name = "INSOFT s.r.o.", email = "helpdesk@insoft.cz"}
@@ -0,0 +1,193 @@
1
+ """Test ingest(): storing external data with invalid values repaired."""
2
+
3
+ from typing import Annotated, Optional, Union
4
+
5
+ import msgspec
6
+ import pytest
7
+ from msgspec import Meta
8
+
9
+ from brainlessdb import BrainlessBucket, BrainlessDB, BrainlessDBFeat, BrainlessStruct, Repair
10
+ from tests.conftest import SipConfigV1
11
+
12
+
13
+ class PeerV1(BrainlessStruct):
14
+ port: Annotated[int, Meta(gt=0, lt=65536)] = 5060
15
+ addr: Optional[str] = None
16
+
17
+
18
+ class SpeedDialV1(BrainlessStruct):
19
+ pos: int
20
+ number: str
21
+
22
+
23
+ class ExtensionLineV1(BrainlessStruct):
24
+ pos: int
25
+ peer: PeerV1 = msgspec.field(default_factory=PeerV1)
26
+
27
+
28
+ class PhoneV1(BrainlessBucket, kw_only=True):
29
+ @staticmethod
30
+ def allows_app(app: str) -> bool:
31
+ return True
32
+
33
+ id: Annotated[int, Meta(extra={"brainlessdb_flags": BrainlessDBFeat.UNIQUE})]
34
+ name: str
35
+ mac: Optional[Annotated[str, Meta(pattern="^[0-9A-F]{12}$")]] = None
36
+ vlan: Annotated[int, Meta(ge=0, lt=4096)] = 0
37
+ nat: bool = False
38
+ peer: PeerV1 = msgspec.field(default_factory=PeerV1)
39
+ lines: list[Union[SpeedDialV1, ExtensionLineV1]] = []
40
+ data: dict[str, int] = {}
41
+
42
+
43
+ @pytest.fixture
44
+ async def phones():
45
+ db = BrainlessDB(namespace="test")
46
+ coll = db.collection(PhoneV1)
47
+ await db.start()
48
+ return coll
49
+
50
+
51
+ def actions(repairs: list[Repair]) -> dict[tuple, str]:
52
+ return {r.path: r.action for r in repairs}
53
+
54
+
55
+ class TestCollectionIngestAdd:
56
+ @pytest.mark.asyncio
57
+ async def test_valid_data_has_no_repairs(self, phones):
58
+ item, repairs = phones.ingest({"id": 1, "name": "a", "vlan": 10}, key="id")
59
+ assert repairs == []
60
+ assert item.vlan == 10
61
+ assert phones.find(id=1) is item
62
+
63
+ @pytest.mark.asyncio
64
+ async def test_reports_unknown_keys(self, phones):
65
+ item, repairs = phones.ingest({"id": 1, "name": "a", "unknown": 1}, key="id")
66
+ assert item.name == "a"
67
+ assert actions(repairs) == {("unknown",): "ignored"}
68
+
69
+ @pytest.mark.asyncio
70
+ async def test_invalid_field_gets_default(self, phones):
71
+ item, repairs = phones.ingest({"id": 1, "name": "a", "mac": "nope", "vlan": 5000}, key="id")
72
+ assert item.mac is None
73
+ assert item.vlan == 0
74
+ assert actions(repairs) == {("mac",): "default", ("vlan",): "default"}
75
+
76
+ @pytest.mark.asyncio
77
+ async def test_values_are_coerced(self, phones):
78
+ item, repairs = phones.ingest({"id": "1", "name": "a", "vlan": "10", "nat": "true"}, key="id")
79
+ assert (item.id, item.vlan, item.nat) == (1, 10, True)
80
+ assert repairs == []
81
+
82
+ @pytest.mark.asyncio
83
+ async def test_invalid_list_item_is_removed(self, phones):
84
+ lines = [
85
+ {"TYPE": "SpeedDialV1", "pos": 0, "number": "100"},
86
+ {"TYPE": "SpeedDialV1", "pos": 1}, # missing required number
87
+ {"TYPE": "ExtensionLineV1", "pos": 2, "peer": {"port": 0}},
88
+ ]
89
+ item, repairs = phones.ingest({"id": 1, "name": "a", "lines": lines}, key="id")
90
+ assert [line.pos for line in item.lines] == [0, 2]
91
+ assert item.lines[1].peer.port == 5060
92
+ assert actions(repairs) == {("lines", 1): "removed", ("lines", 2, "peer", "port"): "default"}
93
+
94
+ @pytest.mark.asyncio
95
+ async def test_invalid_dict_entry_is_removed(self, phones):
96
+ item, repairs = phones.ingest({"id": 1, "name": "a", "data": {"a": 1, "b": "x"}}, key="id")
97
+ assert item.data == {"a": 1}
98
+ assert actions(repairs) == {("data", "b"): "removed"}
99
+
100
+ @pytest.mark.asyncio
101
+ async def test_invalid_required_field_raises(self, phones):
102
+ with pytest.raises(msgspec.ValidationError):
103
+ phones.ingest({"id": 1, "name": None}, key="id")
104
+ assert len(phones) == 0
105
+
106
+ @pytest.mark.asyncio
107
+ async def test_repair_does_not_record_value(self, phones):
108
+ _, repairs = phones.ingest({"id": 1, "name": "a", "mac": "secret-value"}, key="id")
109
+ assert "secret-value" not in repr(repairs)
110
+
111
+ @pytest.mark.asyncio
112
+ async def test_input_is_not_mutated(self, phones):
113
+ data = {"id": 1, "name": "a", "mac": "nope", "lines": [{"TYPE": "SpeedDialV1", "pos": 1}]}
114
+ phones.ingest(data, key="id")
115
+ assert data == {"id": 1, "name": "a", "mac": "nope", "lines": [{"TYPE": "SpeedDialV1", "pos": 1}]}
116
+
117
+
118
+ class TestCollectionIngestUpdate:
119
+ @pytest.mark.asyncio
120
+ async def test_invalid_field_keeps_current(self, phones):
121
+ phones.ingest({"id": 1, "name": "a", "vlan": 10, "mac": "AABBCCDDEEFF"}, key="id")
122
+ item, repairs = phones.ingest({"id": 1, "name": None, "vlan": "x", "nat": True}, key="id")
123
+ assert (item.name, item.vlan, item.nat, item.mac) == ("a", 10, True, "AABBCCDDEEFF")
124
+ assert actions(repairs) == {("name",): "kept", ("vlan",): "kept"}
125
+
126
+ @pytest.mark.asyncio
127
+ async def test_nested_struct_keeps_current(self, phones):
128
+ phones.ingest({"id": 1, "name": "a", "peer": {"port": 5070, "addr": "10.0.0.1"}}, key="id")
129
+ item, repairs = phones.ingest({"id": 1, "peer": {"port": -1, "addr": "10.0.0.2"}}, key="id")
130
+ assert (item.peer.port, item.peer.addr) == (5070, "10.0.0.2")
131
+ assert actions(repairs) == {("peer", "port"): "kept"}
132
+
133
+ @pytest.mark.asyncio
134
+ async def test_returns_stored_item(self, phones):
135
+ phones.ingest({"id": 1, "name": "a"}, key="id")
136
+ item, _ = phones.ingest({"id": 1, "name": "b"}, key="id")
137
+ assert item is phones.find(id=1)
138
+ assert item.name == "b"
139
+
140
+ @pytest.mark.asyncio
141
+ async def test_key_must_be_unique_field(self, phones):
142
+ with pytest.raises(ValueError, match="unique"):
143
+ phones.ingest({"id": 1, "name": "a"}, key="name")
144
+
145
+
146
+ class TestItemIngest:
147
+ @pytest.mark.asyncio
148
+ async def test_partial_update_keeps_other_fields(self, phones):
149
+ item, _ = phones.ingest({"id": 1, "name": "a", "vlan": 10}, key="id")
150
+ repairs = item.ingest({"nat": True, "vlan": 99999})
151
+ stored = phones.find(id=1)
152
+ assert (stored.name, stored.vlan, stored.nat) == ("a", 10, True)
153
+ assert actions(repairs) == {("vlan",): "kept"}
154
+
155
+ @pytest.mark.asyncio
156
+ async def test_reports_unknown_keys(self, phones):
157
+ item, _ = phones.ingest({"id": 1, "name": "a"}, key="id")
158
+ assert actions(item.ingest({"unknown": 1})) == {("unknown",): "ignored"}
159
+ assert phones.find(id=1).name == "a"
160
+
161
+
162
+ class TestRegistryIngest:
163
+ @pytest.mark.asyncio
164
+ async def test_invalid_value_keeps_current(self):
165
+ db = BrainlessDB(namespace="test")
166
+ sip = db.registry(SipConfigV1)
167
+ await db.start()
168
+
169
+ repairs = sip.ingest({"host": None, "port": "5070"})
170
+ assert (sip.host, sip.port) == ("127.0.0.1", 5070)
171
+ assert actions(repairs) == {("host",): "kept"}
172
+ await db.stop()
173
+
174
+ @pytest.mark.asyncio
175
+ async def test_reports_unknown_keys(self):
176
+ db = BrainlessDB(namespace="test")
177
+ sip = db.registry(SipConfigV1)
178
+ await db.start()
179
+
180
+ repairs = sip.ingest({"hots": "10.0.0.1", "port": 5070})
181
+ assert (sip.host, sip.port) == ("127.0.0.1", 5070)
182
+ assert actions(repairs) == {("hots",): "ignored"}
183
+ await db.stop()
184
+
185
+ @pytest.mark.asyncio
186
+ async def test_marks_dirty(self):
187
+ db = BrainlessDB(namespace="test")
188
+ sip = db.registry(SipConfigV1)
189
+ await db.start()
190
+
191
+ sip.ingest({"port": 5070})
192
+ assert db._registries["sipconfigv1"]._dirty
193
+ await db.stop()
@@ -17,7 +17,7 @@ wheels = [
17
17
 
18
18
  [[package]]
19
19
  name = "brainlessdb"
20
- version = "0.17.3"
20
+ version = "1.1.0"
21
21
  source = { editable = "." }
22
22
  dependencies = [
23
23
  { name = "msgspec" },
@@ -1 +0,0 @@
1
- 3.14
File without changes
File without changes
File without changes