lorebase-core 0.2.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. lorebase_core/__init__.py +1 -0
  2. lorebase_core/canonical/__init__.py +35 -0
  3. lorebase_core/canonical/hashing.py +128 -0
  4. lorebase_core/canonical/jcs.py +162 -0
  5. lorebase_core/expr/__init__.py +118 -0
  6. lorebase_core/expr/ast.py +443 -0
  7. lorebase_core/expr/checker.py +428 -0
  8. lorebase_core/expr/compiler.py +87 -0
  9. lorebase_core/expr/context.py +82 -0
  10. lorebase_core/expr/derivations.py +306 -0
  11. lorebase_core/expr/environment.py +176 -0
  12. lorebase_core/expr/errors.py +98 -0
  13. lorebase_core/expr/evaluator.py +414 -0
  14. lorebase_core/expr/functions.py +898 -0
  15. lorebase_core/expr/limits.py +52 -0
  16. lorebase_core/expr/operations.py +479 -0
  17. lorebase_core/expr/ops.py +243 -0
  18. lorebase_core/expr/resolver.py +96 -0
  19. lorebase_core/expr/rows.py +148 -0
  20. lorebase_core/expr/sensitivity.py +337 -0
  21. lorebase_core/expr/types.py +227 -0
  22. lorebase_core/expr/validation.py +51 -0
  23. lorebase_core/expr/values.py +169 -0
  24. lorebase_core/expr/visibility.py +363 -0
  25. lorebase_core/infer/__init__.py +36 -0
  26. lorebase_core/infer/columns.py +621 -0
  27. lorebase_core/infer/engine.py +546 -0
  28. lorebase_core/infer/keys.py +164 -0
  29. lorebase_core/infer/names.py +145 -0
  30. lorebase_core/infer/options.py +290 -0
  31. lorebase_core/infer/profile.py +54 -0
  32. lorebase_core/infer/proposal.py +122 -0
  33. lorebase_core/infer/references.py +314 -0
  34. lorebase_core/infer/suggestions.py +731 -0
  35. lorebase_core/infer/tables.py +81 -0
  36. lorebase_core/merge/__init__.py +89 -0
  37. lorebase_core/merge/conflicts.py +262 -0
  38. lorebase_core/merge/consistency.py +440 -0
  39. lorebase_core/merge/data_merge.py +522 -0
  40. lorebase_core/merge/engine.py +476 -0
  41. lorebase_core/merge/index.py +100 -0
  42. lorebase_core/merge/objects.py +204 -0
  43. lorebase_core/merge/policies.py +322 -0
  44. lorebase_core/merge/resolve.py +321 -0
  45. lorebase_core/merge/schema_merge.py +429 -0
  46. lorebase_core/merge/state.py +443 -0
  47. lorebase_core/model/__init__.py +245 -0
  48. lorebase_core/model/attribute.py +72 -0
  49. lorebase_core/model/base.py +61 -0
  50. lorebase_core/model/derivation.py +95 -0
  51. lorebase_core/model/domain.py +71 -0
  52. lorebase_core/model/embedding.py +193 -0
  53. lorebase_core/model/ids.py +78 -0
  54. lorebase_core/model/kind.py +85 -0
  55. lorebase_core/model/lineage.py +114 -0
  56. lorebase_core/model/operation.py +161 -0
  57. lorebase_core/model/paths.py +141 -0
  58. lorebase_core/model/record.py +127 -0
  59. lorebase_core/model/relation.py +83 -0
  60. lorebase_core/model/schema.py +219 -0
  61. lorebase_core/model/source.py +204 -0
  62. lorebase_core/model/trait.py +37 -0
  63. lorebase_core/model/vault.py +43 -0
  64. lorebase_core/model/yaml_schema.py +1058 -0
  65. lorebase_core/query/__init__.py +97 -0
  66. lorebase_core/query/dsl.py +227 -0
  67. lorebase_core/query/resolve.py +794 -0
  68. lorebase_core/revision/__init__.py +59 -0
  69. lorebase_core/revision/ancestry.py +272 -0
  70. lorebase_core/revision/changeset.py +62 -0
  71. lorebase_core/revision/revision.py +121 -0
  72. lorebase_core/tree/__init__.py +76 -0
  73. lorebase_core/tree/diff.py +119 -0
  74. lorebase_core/tree/hamt.py +333 -0
  75. lorebase_core/tree/node.py +280 -0
  76. lorebase_core/tree/params.py +33 -0
  77. lorebase_core/tree/root.py +70 -0
  78. lorebase_core/tree/store.py +98 -0
  79. lorebase_core/types/__init__.py +71 -0
  80. lorebase_core/types/casts.py +271 -0
  81. lorebase_core/types/constraints.py +299 -0
  82. lorebase_core/types/encoding.py +538 -0
  83. lorebase_core/types/errors.py +41 -0
  84. lorebase_core/types/matching.py +40 -0
  85. lorebase_core/types/types.py +187 -0
  86. lorebase_core/validate/__init__.py +49 -0
  87. lorebase_core/validate/changes.py +70 -0
  88. lorebase_core/validate/changeset.py +551 -0
  89. lorebase_core/validate/data_rules.py +291 -0
  90. lorebase_core/validate/issues.py +123 -0
  91. lorebase_core/validate/schema_rules.py +1304 -0
  92. lorebase_core-0.2.3.dist-info/METADATA +9 -0
  93. lorebase_core-0.2.3.dist-info/RECORD +94 -0
  94. lorebase_core-0.2.3.dist-info/WHEEL +4 -0
@@ -0,0 +1,1304 @@
1
+ """Static schema validation (design 5.1 to 5.9, invariant I1: the schema is internally consistent).
2
+
3
+ ``validate_schema`` returns every problem found instead of stopping at the first one, so
4
+ that editors can show them all. Issues of severity ``error`` make a schema invalid.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from collections import defaultdict
10
+ from collections.abc import Iterable
11
+ from dataclasses import dataclass, field
12
+ from typing import TYPE_CHECKING, NamedTuple
13
+
14
+ from lorebase_core.model.attribute import Attribute, Sensitivity
15
+ from lorebase_core.model.base import ElementStatus, NameStyle, is_valid_api_name
16
+ from lorebase_core.model.derivation import (
17
+ AggregateDerivation,
18
+ AggregateFunction,
19
+ Derivation,
20
+ ExpressionDerivation,
21
+ )
22
+ from lorebase_core.model.embedding import template_problems
23
+ from lorebase_core.model.kind import Kind
24
+ from lorebase_core.model.operation import (
25
+ SIDE_EFFECT_MAX_LENGTH,
26
+ CreateLinkStep,
27
+ CreateRecordStep,
28
+ DeleteLinkStep,
29
+ DeleteRecordStep,
30
+ Operation,
31
+ UpdateRecordStep,
32
+ builtin_link_operation_names,
33
+ builtin_operation_names,
34
+ )
35
+ from lorebase_core.model.relation import Relation, RelationBacking
36
+ from lorebase_core.model.schema import Schema
37
+ from lorebase_core.model.source import SECRET_REFERENCE_PREFIX, Source, SourceType, SyncMode
38
+ from lorebase_core.types import (
39
+ TEXTUAL_TYPES,
40
+ AttributeType,
41
+ CastRule,
42
+ EncodingError,
43
+ TypeSpec,
44
+ cast_rule,
45
+ check_constraints,
46
+ constraint_problems,
47
+ encode_value,
48
+ )
49
+ from lorebase_core.validate.issues import Issue, IssueCode, PathPart, Severity
50
+
51
+ if TYPE_CHECKING:
52
+ from lorebase_core.expr.limits import ExprLimits
53
+
54
+ #: Attribute api names reserved for Record envelopes in the API and the query DSL.
55
+ DEFAULT_RESERVED_ATTRIBUTE_NAMES = frozenset({"key", "kind", "ref", "revision"})
56
+
57
+ #: Api name offered instead when a Kind Attribute takes a reserved one.
58
+ RESERVED_NAME_SUGGESTIONS = {
59
+ "key": "code",
60
+ "kind": "category",
61
+ "ref": "reference",
62
+ "revision": "version",
63
+ }
64
+
65
+
66
+ def reserved_name_message(name: str, reserved: frozenset[str]) -> str:
67
+ """Why `name` is refused and what to use instead."""
68
+ suggestion = RESERVED_NAME_SUGGESTIONS.get(name, f"{name}_value")
69
+ return (
70
+ f"`{name}` is a reserved name, choose another (e.g. `{suggestion}`); Kind Attributes "
71
+ f"cannot be named {', '.join(f'`{n}`' for n in sorted(reserved))}, which name the "
72
+ "Record envelope in the API and the query language"
73
+ )
74
+
75
+
76
+ #: Types usable as primary key components (rendered as stable key text).
77
+ KEYABLE_TYPES = frozenset(
78
+ {
79
+ AttributeType.STRING,
80
+ AttributeType.INTEGER,
81
+ AttributeType.DECIMAL,
82
+ AttributeType.DATE,
83
+ AttributeType.TIMESTAMP,
84
+ AttributeType.ENUM,
85
+ AttributeType.URI,
86
+ AttributeType.REF,
87
+ }
88
+ )
89
+
90
+ #: Types that cannot be ``unique``: no reliable equality (float) or not scalar.
91
+ NON_UNIQUE_TYPES = frozenset(
92
+ {AttributeType.FLOAT, AttributeType.JSON, AttributeType.GEO_POINT, AttributeType.ARRAY}
93
+ )
94
+
95
+ #: Types that cannot be ``indexed``: opaque values (design 5.4).
96
+ NON_INDEXED_TYPES = frozenset({AttributeType.JSON})
97
+
98
+ #: Types that full-text search can read.
99
+ SEARCHABLE_TYPES = TEXTUAL_TYPES | {AttributeType.ENUM}
100
+
101
+ #: Aggregates that need a numeric source Attribute.
102
+ _NUMERIC_AGGREGATES = frozenset({AggregateFunction.SUM, AggregateFunction.AVG})
103
+ _NUMERIC_TYPES = frozenset({AttributeType.INTEGER, AttributeType.DECIMAL, AttributeType.FLOAT})
104
+ _QUERY_SOURCE_TYPES = frozenset(
105
+ {SourceType.POSTGRES_QUERY, SourceType.MYSQL_QUERY, SourceType.MSSQL_QUERY}
106
+ )
107
+ #: Source fields that only some Source types read.
108
+ _SOURCE_TYPE_FIELDS: dict[str, tuple[SourceType, ...]] = {
109
+ "records_path": (SourceType.JSON, SourceType.HTTP_JSON),
110
+ "flatten_separator": (
111
+ SourceType.JSON,
112
+ SourceType.JSONL,
113
+ SourceType.PARQUET,
114
+ SourceType.HTTP_JSON,
115
+ ),
116
+ "http": (SourceType.HTTP_JSON,),
117
+ }
118
+
119
+
120
+ def _where(path: tuple[PathPart, ...]) -> str:
121
+ return "/".join(map(str, path))
122
+
123
+
124
+ @dataclass(frozen=True)
125
+ class SchemaRules:
126
+ """Options of static validation.
127
+
128
+ ``allow_expressions`` gates Operations and Derivations that carry CEL expressions. When
129
+ false they are refused (``schema.expressions_disabled``); when true every expression is
130
+ parsed and type checked (``schema.expression_invalid``) within ``expression_limits``
131
+ (the ``ExprLimits`` defaults when unset).
132
+ """
133
+
134
+ reserved_attribute_names: frozenset[str] = DEFAULT_RESERVED_ATTRIBUTE_NAMES
135
+ allow_expressions: bool = False
136
+ expression_limits: ExprLimits | None = None
137
+
138
+
139
+ class Refs(NamedTuple):
140
+ kind: str | None
141
+ relation: str | None
142
+ attribute: str | None
143
+
144
+
145
+ @dataclass
146
+ class _Checker:
147
+ schema: Schema
148
+ rules: SchemaRules
149
+ issues: list[Issue] = field(default_factory=list)
150
+
151
+ def add(
152
+ self,
153
+ code: IssueCode,
154
+ path: tuple[PathPart, ...],
155
+ message: str,
156
+ *,
157
+ severity: Severity = Severity.ERROR,
158
+ kind: str | None = None,
159
+ relation: str | None = None,
160
+ attribute: str | None = None,
161
+ refs: Refs | None = None,
162
+ ) -> None:
163
+ if refs is not None:
164
+ kind, relation, attribute = refs
165
+ self.issues.append(
166
+ Issue(
167
+ code=code,
168
+ severity=severity,
169
+ path=path,
170
+ message=message,
171
+ kind=kind,
172
+ relation=relation,
173
+ attribute=attribute,
174
+ )
175
+ )
176
+
177
+ # Names -------------------------------------------------------------------
178
+
179
+ def check_name(self, name: str, style: NameStyle, path: tuple[PathPart, ...]) -> None:
180
+ if not is_valid_api_name(name, style):
181
+ self.add(
182
+ IssueCode.SCHEMA_API_NAME_INVALID, path, f"api name {name!r} is not {style.value}"
183
+ )
184
+
185
+ def check_unique(self, entries: Iterable[tuple[str, tuple[PathPart, ...]]], scope: str) -> None:
186
+ seen: dict[str, tuple[PathPart, ...]] = {}
187
+ for name, path in entries:
188
+ if name in seen:
189
+ self.add(
190
+ IssueCode.SCHEMA_API_NAME_DUPLICATE,
191
+ path,
192
+ f"api name {name!r} is already used in {scope} at {_where(seen[name])}",
193
+ )
194
+ else:
195
+ seen[name] = path
196
+
197
+ def check_names(self) -> None:
198
+ schema = self.schema
199
+ type_names: list[tuple[str, tuple[PathPart, ...]]] = []
200
+ for kind in schema.kinds.values():
201
+ path: tuple[PathPart, ...] = ("kinds", kind.id, "api_name")
202
+ self.check_name(kind.api_name, NameStyle.PASCAL, path)
203
+ type_names.append((kind.api_name, path))
204
+ self.check_attribute_names(kind.attributes, ("kinds", kind.id), reserved=True)
205
+ for trait in schema.traits.values():
206
+ path = ("traits", trait.id, "api_name")
207
+ self.check_name(trait.api_name, NameStyle.PASCAL, path)
208
+ type_names.append((trait.api_name, path))
209
+ trait_attribute_names = []
210
+ for attribute in trait.attributes.values():
211
+ attribute_path = ("traits", trait.id, "attributes", attribute.id, "api_name")
212
+ self.check_name(attribute.api_name, NameStyle.SNAKE, attribute_path)
213
+ trait_attribute_names.append((attribute.api_name, attribute_path))
214
+ self.check_unique(trait_attribute_names, f"Trait {trait.api_name}")
215
+ self.check_unique(type_names, "Kinds and Traits")
216
+
217
+ relation_names: list[tuple[str, tuple[PathPart, ...]]] = []
218
+ for relation in schema.relations.values():
219
+ forward = ("relations", relation.id, "api_name")
220
+ inverse = ("relations", relation.id, "inverse_api_name")
221
+ self.check_name(relation.api_name, NameStyle.SNAKE, forward)
222
+ self.check_name(relation.inverse_api_name, NameStyle.SNAKE, inverse)
223
+ relation_names.append((relation.api_name, forward))
224
+ symmetric = (
225
+ relation.inverse_api_name == relation.api_name
226
+ and relation.from_kind == relation.to_kind
227
+ )
228
+ if not symmetric:
229
+ relation_names.append((relation.inverse_api_name, inverse))
230
+ self.check_attribute_names(
231
+ relation.attributes, ("relations", relation.id), reserved=False
232
+ )
233
+ for derivation in schema.derivations.values():
234
+ definition = derivation.definition
235
+ if definition.variant == "path":
236
+ relation_names.append(
237
+ (derivation.api_name, ("derivations", derivation.id, "api_name"))
238
+ )
239
+ if definition.inverse_api_name is not None:
240
+ path = ("derivations", derivation.id, "definition", "inverse_api_name")
241
+ self.check_name(definition.inverse_api_name, NameStyle.SNAKE, path)
242
+ relation_names.append((definition.inverse_api_name, path))
243
+ self.check_unique(relation_names, "Relations")
244
+
245
+ builtin = [
246
+ (name, ("kinds", kind.id, "operations_builtin"))
247
+ for kind in schema.kinds.values()
248
+ for name in builtin_operation_names(kind)
249
+ ] + [
250
+ (name, ("relations", relation.id))
251
+ for relation in schema.relations.values()
252
+ if relation.backing is RelationBacking.EXPLICIT
253
+ for name in builtin_link_operation_names(relation)
254
+ ]
255
+ operation_names: list[tuple[str, tuple[PathPart, ...]]] = []
256
+ for operation in schema.operations.values():
257
+ path = ("operations", operation.id, "api_name")
258
+ self.check_name(operation.api_name, NameStyle.SNAKE, path)
259
+ operation_names.append((operation.api_name, path))
260
+ parameter_names = []
261
+ for index, parameter in enumerate(operation.parameters):
262
+ parameter_path = ("operations", operation.id, "parameters", index, "name")
263
+ self.check_name(parameter.name, NameStyle.SNAKE, parameter_path)
264
+ parameter_names.append((parameter.name, parameter_path))
265
+ self.check_unique(parameter_names, f"Operation {operation.api_name}")
266
+ side_effects = []
267
+ for index, name in enumerate(operation.side_effects):
268
+ effect_path = ("operations", operation.id, "side_effects", index)
269
+ self.check_name(name, NameStyle.SNAKE, effect_path)
270
+ if len(name) > SIDE_EFFECT_MAX_LENGTH:
271
+ self.add(
272
+ IssueCode.SCHEMA_API_NAME_INVALID,
273
+ effect_path,
274
+ f"side effect {name!r} is longer than {SIDE_EFFECT_MAX_LENGTH} characters",
275
+ )
276
+ side_effects.append((name, effect_path))
277
+ self.check_unique(side_effects, f"the side effects of {operation.api_name}")
278
+ builtin_names = {name for name, _ in builtin}
279
+ for name, path in operation_names:
280
+ if name in builtin_names:
281
+ self.add(
282
+ IssueCode.SCHEMA_API_NAME_DUPLICATE,
283
+ path,
284
+ f"{name!r} is the name of a built-in Operation",
285
+ )
286
+ self.check_unique(operation_names, "Operations")
287
+
288
+ derivation_names = []
289
+ for derivation in schema.derivations.values():
290
+ path = ("derivations", derivation.id, "api_name")
291
+ self.check_name(derivation.api_name, NameStyle.SNAKE, path)
292
+ derivation_names.append((derivation.api_name, path))
293
+ self.check_unique(derivation_names, "Derivations")
294
+
295
+ source_names = []
296
+ for source in schema.sources.values():
297
+ path = ("sources", source.id, "api_name")
298
+ self.check_name(source.api_name, NameStyle.DOTTED_SNAKE, path)
299
+ source_names.append((source.api_name, path))
300
+ self.check_unique(source_names, "Sources")
301
+
302
+ def check_attribute_names(
303
+ self, attributes: dict[str, Attribute], owner: tuple[PathPart, ...], *, reserved: bool
304
+ ) -> None:
305
+ names = []
306
+ for attribute in attributes.values():
307
+ path = (*owner, "attributes", attribute.id, "api_name")
308
+ self.check_name(attribute.api_name, NameStyle.SNAKE, path)
309
+ if reserved and attribute.api_name in self.rules.reserved_attribute_names:
310
+ self.add(
311
+ IssueCode.SCHEMA_API_NAME_RESERVED,
312
+ path,
313
+ reserved_name_message(attribute.api_name, self.rules.reserved_attribute_names),
314
+ )
315
+ names.append((attribute.api_name, path))
316
+ self.check_unique(names, "/".join(map(str, owner)))
317
+
318
+ def check_attribute_ids(self) -> None:
319
+ owners: dict[str, tuple[PathPart, ...]] = {}
320
+ for collection, elements in (
321
+ ("kinds", self.schema.kinds),
322
+ ("relations", self.schema.relations),
323
+ ("traits", self.schema.traits),
324
+ ):
325
+ for element in elements.values():
326
+ for attribute_id in element.attributes:
327
+ path = (collection, element.id, "attributes", attribute_id)
328
+ if attribute_id in owners:
329
+ self.add(
330
+ IssueCode.SCHEMA_ID_DUPLICATE,
331
+ path,
332
+ f"attribute id {attribute_id} is also declared at "
333
+ f"{_where(owners[attribute_id])}",
334
+ attribute=attribute_id,
335
+ )
336
+ else:
337
+ owners[attribute_id] = path
338
+
339
+ # Types and Attributes --------------------------------------------------------
340
+
341
+ def check_type_references(self, spec: TypeSpec, path: tuple[PathPart, ...]) -> None:
342
+ scalar = spec.scalar
343
+ if scalar.kind_ref is not None and scalar.kind_ref not in self.schema.kinds:
344
+ self.add(
345
+ IssueCode.SCHEMA_TYPE_REFERENCE_UNKNOWN,
346
+ path,
347
+ f"ref type names unknown Kind {scalar.kind_ref} "
348
+ "(Kinds of other Domains cannot be referenced)",
349
+ )
350
+
351
+ def check_attribute(
352
+ self,
353
+ attribute: Attribute,
354
+ owner: tuple[PathPart, ...],
355
+ *,
356
+ kind_id: str | None,
357
+ relation_id: str | None,
358
+ active: bool,
359
+ ) -> None:
360
+ path = (*owner, "attributes", attribute.id)
361
+ refs = Refs(kind_id, relation_id, attribute.id)
362
+ spec = attribute.type
363
+ if active and not attribute.description.strip():
364
+ self.add(
365
+ IssueCode.SCHEMA_DESCRIPTION_MISSING,
366
+ (*path, "description"),
367
+ f"attribute {attribute.api_name!r} of an active element needs a description",
368
+ refs=refs,
369
+ )
370
+ self.check_type_references(spec, (*path, "type"))
371
+ for name, message in constraint_problems(spec, attribute.constraints):
372
+ self.add(
373
+ IssueCode.SCHEMA_CONSTRAINT_INVALID,
374
+ (*path, "constraints", name.value),
375
+ message,
376
+ refs=refs,
377
+ )
378
+
379
+ if attribute.default is not None:
380
+ try:
381
+ canonical = encode_value(spec, attribute.default)
382
+ except EncodingError as exc:
383
+ self.add(
384
+ IssueCode.SCHEMA_DEFAULT_INVALID,
385
+ (*path, "default"),
386
+ f"default: {exc.message}",
387
+ refs=refs,
388
+ )
389
+ else:
390
+ if canonical != attribute.default:
391
+ self.add(
392
+ IssueCode.SCHEMA_DEFAULT_INVALID,
393
+ (*path, "default"),
394
+ f"default must be in canonical form: {canonical!r}",
395
+ refs=refs,
396
+ )
397
+ elif not constraint_problems(spec, attribute.constraints):
398
+ for violation in check_constraints(spec, attribute.constraints, canonical):
399
+ self.add(
400
+ IssueCode.SCHEMA_DEFAULT_INVALID,
401
+ (*path, "default"),
402
+ f"default violates {violation.constraint.value}: {violation.message}",
403
+ refs=refs,
404
+ )
405
+ if attribute.is_derived:
406
+ self.add(
407
+ IssueCode.SCHEMA_DEFAULT_CONFLICT,
408
+ (*path, "default"),
409
+ "derived attributes are computed and cannot have a default",
410
+ refs=refs,
411
+ )
412
+ if attribute.unique:
413
+ self.add(
414
+ IssueCode.SCHEMA_DEFAULT_CONFLICT,
415
+ (*path, "default"),
416
+ "a default on a unique attribute is shared by every Record omitting the value",
417
+ severity=Severity.WARNING,
418
+ refs=refs,
419
+ )
420
+ if attribute.is_derived and attribute.required:
421
+ self.add(
422
+ IssueCode.SCHEMA_DEFAULT_CONFLICT,
423
+ (*path, "required"),
424
+ "derived attributes are computed and cannot be required",
425
+ refs=refs,
426
+ )
427
+
428
+ if attribute.unique and relation_id is not None:
429
+ self.add(
430
+ IssueCode.SCHEMA_ATTRIBUTE_FLAG_INVALID,
431
+ (*path, "unique"),
432
+ "link attributes cannot be unique",
433
+ refs=refs,
434
+ )
435
+ elif attribute.unique and spec.name in NON_UNIQUE_TYPES:
436
+ self.add(
437
+ IssueCode.SCHEMA_ATTRIBUTE_FLAG_INVALID,
438
+ (*path, "unique"),
439
+ f"{spec.describe()} attributes cannot be unique",
440
+ refs=refs,
441
+ )
442
+ if attribute.indexed and spec.scalar.name in NON_INDEXED_TYPES:
443
+ self.add(
444
+ IssueCode.SCHEMA_ATTRIBUTE_FLAG_INVALID,
445
+ (*path, "indexed"),
446
+ f"{spec.describe()} attributes cannot be indexed",
447
+ refs=refs,
448
+ )
449
+ if attribute.searchable and spec.scalar.name not in SEARCHABLE_TYPES:
450
+ self.add(
451
+ IssueCode.SCHEMA_ATTRIBUTE_FLAG_INVALID,
452
+ (*path, "searchable"),
453
+ f"{spec.describe()} attributes cannot be full-text searchable",
454
+ refs=refs,
455
+ )
456
+ if attribute.derivation_id is not None:
457
+ if relation_id is not None:
458
+ self.add(
459
+ IssueCode.SCHEMA_ATTRIBUTE_FLAG_INVALID,
460
+ (*path, "derivation_id"),
461
+ "link attributes cannot be derived",
462
+ refs=refs,
463
+ )
464
+ else:
465
+ derivation = self.schema.derivations.get(attribute.derivation_id)
466
+ if derivation is None:
467
+ self.add(
468
+ IssueCode.SCHEMA_DERIVATION_REFERENCE_UNKNOWN,
469
+ (*path, "derivation_id"),
470
+ f"unknown Derivation {attribute.derivation_id}",
471
+ refs=refs,
472
+ )
473
+ elif derivation.target != (kind_id, attribute.id):
474
+ self.add(
475
+ IssueCode.SCHEMA_DERIVATION_INVALID,
476
+ (*path, "derivation_id"),
477
+ f"Derivation {derivation.api_name!r} does not compute this attribute",
478
+ refs=refs,
479
+ )
480
+ self.check_sensitive(attribute, path, refs, relation_id=relation_id)
481
+ if attribute.authority is not None and attribute.authority not in self.schema.sources:
482
+ self.add(
483
+ IssueCode.SCHEMA_AUTHORITY_UNKNOWN,
484
+ (*path, "authority"),
485
+ f"unknown Source {attribute.authority}",
486
+ refs=refs,
487
+ )
488
+
489
+ def check_sensitive(
490
+ self,
491
+ attribute: Attribute,
492
+ path: tuple[PathPart, ...],
493
+ refs: Refs,
494
+ *,
495
+ relation_id: str | None,
496
+ ) -> None:
497
+ """Vault storage (design 13.5) and what sensitivity tags take out of search."""
498
+ if attribute.is_sensitive and attribute.searchable:
499
+ self.add(
500
+ IssueCode.SCHEMA_SENSITIVE_ATTRIBUTE,
501
+ (*path, "searchable"),
502
+ f"sensitive attribute {attribute.api_name!r} is left out of full-text search",
503
+ severity=Severity.WARNING,
504
+ refs=refs,
505
+ )
506
+ if attribute.vault is None:
507
+ return
508
+ problems: list[tuple[str, str]] = []
509
+ if Sensitivity.PII not in attribute.sensitivity:
510
+ problems.append(("sensitivity", "vault storage needs the pii sensitivity tag"))
511
+ if relation_id is not None:
512
+ problems.append(("vault", "link attributes cannot use vault storage"))
513
+ if attribute.is_derived:
514
+ problems.append(("vault", "derived attributes cannot use vault storage"))
515
+ if attribute.in_vault:
516
+ if attribute.unique:
517
+ problems.append(("unique", "vault attributes cannot be unique"))
518
+ if attribute.indexed:
519
+ problems.append(("indexed", "vault attributes cannot be indexed"))
520
+ if attribute.type.scalar.name is AttributeType.REF:
521
+ problems.append(("type", "references to Records cannot use vault storage"))
522
+ for field_name, message in problems:
523
+ self.add(
524
+ IssueCode.SCHEMA_VAULT_INVALID,
525
+ (*path, field_name),
526
+ f"attribute {attribute.api_name!r}: {message}",
527
+ refs=refs,
528
+ )
529
+
530
+ def check_vault_roles(self, kind: Kind) -> None:
531
+ """Vault attributes hold references, so they cannot key, label or link Records."""
532
+ base: tuple[PathPart, ...] = ("kinds", kind.id)
533
+ roles: list[tuple[str, tuple[PathPart, ...], str]] = [
534
+ (attribute_id, (*base, "primary_key", index), "a primary key attribute")
535
+ for index, attribute_id in enumerate(kind.primary_key)
536
+ ]
537
+ if kind.title_attribute is not None:
538
+ roles.append((kind.title_attribute, (*base, "title_attribute"), "the title attribute"))
539
+ for relation in self.schema.relations.values():
540
+ if relation.from_kind == kind.id and relation.foreign_key_attribute is not None:
541
+ roles.append(
542
+ (
543
+ relation.foreign_key_attribute,
544
+ ("relations", relation.id, "foreign_key_attribute"),
545
+ f"the foreign key of Relation {relation.api_name!r}",
546
+ )
547
+ )
548
+ for attribute_id, path, role in roles:
549
+ attribute = kind.attributes.get(attribute_id)
550
+ if attribute is None:
551
+ continue
552
+ if attribute.in_vault:
553
+ self.add(
554
+ IssueCode.SCHEMA_VAULT_INVALID,
555
+ path,
556
+ f"vault attribute {attribute.api_name!r} cannot be {role}",
557
+ kind=kind.id,
558
+ attribute=attribute_id,
559
+ )
560
+ elif attribute.is_sensitive and path[-1] != "title_attribute":
561
+ # Keys built from these values are shown to every reader of the Kind.
562
+ self.add(
563
+ IssueCode.SCHEMA_SENSITIVE_ATTRIBUTE,
564
+ path,
565
+ f"sensitive attribute {attribute.api_name!r} cannot be {role}: Record "
566
+ "keys are shown without clearance",
567
+ kind=kind.id,
568
+ attribute=attribute_id,
569
+ )
570
+ title = kind.attributes.get(kind.title_attribute or "")
571
+ if title is not None and title.is_sensitive and not title.in_vault:
572
+ self.add(
573
+ IssueCode.SCHEMA_SENSITIVE_ATTRIBUTE,
574
+ (*base, "title_attribute"),
575
+ f"title attribute {title.api_name!r} is sensitive: Record titles stay empty",
576
+ severity=Severity.WARNING,
577
+ kind=kind.id,
578
+ attribute=title.id,
579
+ )
580
+
581
+ # Kinds -----------------------------------------------------------------------
582
+
583
+ def check_kind(self, kind: Kind) -> None:
584
+ base: tuple[PathPart, ...] = ("kinds", kind.id)
585
+ active = kind.status is ElementStatus.ACTIVE
586
+ for kind_attribute in kind.attributes.values():
587
+ self.check_attribute(
588
+ kind_attribute, base, kind_id=kind.id, relation_id=None, active=active
589
+ )
590
+
591
+ seen: set[str] = set()
592
+ for index, attribute_id in enumerate(kind.primary_key):
593
+ path = (*base, "primary_key", index)
594
+ attribute = kind.attributes.get(attribute_id)
595
+ if attribute is None:
596
+ self.add(
597
+ IssueCode.SCHEMA_PRIMARY_KEY_UNKNOWN,
598
+ path,
599
+ f"primary key attribute {attribute_id} is not an attribute of the Kind",
600
+ kind=kind.id,
601
+ attribute=attribute_id,
602
+ )
603
+ continue
604
+ problems = []
605
+ if attribute_id in seen:
606
+ problems.append("is listed twice")
607
+ if not attribute.required:
608
+ problems.append("must be required")
609
+ if attribute.type.name not in KEYABLE_TYPES:
610
+ problems.append(f"cannot be of type {attribute.type.describe()}")
611
+ if attribute.is_derived:
612
+ problems.append("cannot be derived")
613
+ if attribute.default is not None:
614
+ problems.append("cannot have a default")
615
+ seen.add(attribute_id)
616
+ for problem in problems:
617
+ self.add(
618
+ IssueCode.SCHEMA_PRIMARY_KEY_INVALID,
619
+ path,
620
+ f"primary key attribute {attribute.api_name!r} {problem}",
621
+ kind=kind.id,
622
+ attribute=attribute_id,
623
+ )
624
+
625
+ if kind.title_attribute is None:
626
+ if active:
627
+ self.add(
628
+ IssueCode.SCHEMA_TITLE_ATTRIBUTE_MISSING,
629
+ (*base, "title_attribute"),
630
+ f"active Kind {kind.api_name!r} needs a title attribute",
631
+ kind=kind.id,
632
+ )
633
+ elif kind.title_attribute not in kind.attributes:
634
+ self.add(
635
+ IssueCode.SCHEMA_TITLE_ATTRIBUTE_UNKNOWN,
636
+ (*base, "title_attribute"),
637
+ f"title attribute {kind.title_attribute} is not an attribute of the Kind",
638
+ kind=kind.id,
639
+ attribute=kind.title_attribute,
640
+ )
641
+
642
+ for problem in template_problems(kind):
643
+ self.add(
644
+ IssueCode.SCHEMA_EMBEDDING_TEMPLATE_INVALID,
645
+ (*base, "search", "embedding_template"),
646
+ f"embedding template of Kind {kind.api_name!r}: {problem}",
647
+ kind=kind.id,
648
+ )
649
+
650
+ self.check_vault_roles(kind)
651
+ self.check_trait_implementations(kind)
652
+
653
+ def check_trait_implementations(self, kind: Kind) -> None:
654
+ implemented: set[str] = set()
655
+ for index, implementation in enumerate(kind.traits):
656
+ path: tuple[PathPart, ...] = ("kinds", kind.id, "traits", index)
657
+ trait = self.schema.traits.get(implementation.trait_id)
658
+ if trait is None:
659
+ self.add(
660
+ IssueCode.SCHEMA_TRAIT_UNKNOWN,
661
+ (*path, "trait_id"),
662
+ f"unknown Trait {implementation.trait_id}",
663
+ kind=kind.id,
664
+ )
665
+ continue
666
+ if trait.id in implemented:
667
+ self.add(
668
+ IssueCode.SCHEMA_TRAIT_DUPLICATE,
669
+ path,
670
+ f"Trait {trait.api_name!r} is implemented twice",
671
+ kind=kind.id,
672
+ )
673
+ continue
674
+ implemented.add(trait.id)
675
+ mapping = implementation.attribute_map
676
+ for trait_attribute_id in mapping:
677
+ if trait_attribute_id not in trait.attributes:
678
+ self.add(
679
+ IssueCode.SCHEMA_TRAIT_MAPPING_INVALID,
680
+ (*path, "attribute_map", trait_attribute_id),
681
+ f"{trait_attribute_id} is not an attribute of Trait {trait.api_name!r}",
682
+ kind=kind.id,
683
+ )
684
+ for trait_attribute in trait.attributes.values():
685
+ mapping_path = (*path, "attribute_map", trait_attribute.id)
686
+ kind_attribute_id = mapping.get(trait_attribute.id)
687
+ if kind_attribute_id is None:
688
+ self.add(
689
+ IssueCode.SCHEMA_TRAIT_MAPPING_MISSING,
690
+ mapping_path,
691
+ f"Trait attribute {trait_attribute.api_name!r} of {trait.api_name!r} "
692
+ "is not mapped",
693
+ kind=kind.id,
694
+ )
695
+ continue
696
+ kind_attribute = kind.attributes.get(kind_attribute_id)
697
+ if kind_attribute is None:
698
+ self.add(
699
+ IssueCode.SCHEMA_TRAIT_MAPPING_INVALID,
700
+ mapping_path,
701
+ f"{kind_attribute_id} is not an attribute of the Kind",
702
+ kind=kind.id,
703
+ attribute=kind_attribute_id,
704
+ )
705
+ continue
706
+ if cast_rule(kind_attribute.type, trait_attribute.type) is not CastRule.SAFE:
707
+ self.add(
708
+ IssueCode.SCHEMA_TRAIT_MAPPING_TYPE,
709
+ mapping_path,
710
+ f"{kind_attribute.api_name!r} ({kind_attribute.type.describe()}) is not "
711
+ f"compatible with {trait.api_name}.{trait_attribute.api_name} "
712
+ f"({trait_attribute.type.describe()})",
713
+ kind=kind.id,
714
+ attribute=kind_attribute_id,
715
+ )
716
+ if trait_attribute.required and not kind_attribute.required:
717
+ self.add(
718
+ IssueCode.SCHEMA_TRAIT_MAPPING_REQUIRED,
719
+ mapping_path,
720
+ f"{trait.api_name}.{trait_attribute.api_name} is required, "
721
+ f"{kind_attribute.api_name!r} is not",
722
+ kind=kind.id,
723
+ attribute=kind_attribute_id,
724
+ )
725
+
726
+ # Relations -------------------------------------------------------------------
727
+
728
+ def check_relation(self, relation: Relation) -> None:
729
+ base: tuple[PathPart, ...] = ("relations", relation.id)
730
+ for end, kind_id in (("from_kind", relation.from_kind), ("to_kind", relation.to_kind)):
731
+ if kind_id not in self.schema.kinds:
732
+ self.add(
733
+ IssueCode.SCHEMA_RELATION_ENDPOINT_UNKNOWN,
734
+ (*base, end),
735
+ f"{end} {kind_id} is not a Kind of Domain {self.schema.domain.id!r} "
736
+ "(cross-Domain Relations are not supported)",
737
+ relation=relation.id,
738
+ )
739
+ active = relation.status is ElementStatus.ACTIVE
740
+ for link_attribute in relation.attributes.values():
741
+ self.check_attribute(
742
+ link_attribute, base, kind_id=None, relation_id=relation.id, active=active
743
+ )
744
+
745
+ fk_path = (*base, "foreign_key_attribute")
746
+ if relation.backing is RelationBacking.EXPLICIT:
747
+ if relation.foreign_key_attribute is not None:
748
+ self.add(
749
+ IssueCode.SCHEMA_FOREIGN_KEY_INVALID,
750
+ fk_path,
751
+ "explicit Relations have no foreign key attribute",
752
+ relation=relation.id,
753
+ )
754
+ return
755
+ if relation.attributes:
756
+ self.add(
757
+ IssueCode.SCHEMA_FOREIGN_KEY_INVALID,
758
+ (*base, "attributes"),
759
+ "foreign key Relations cannot carry link attributes",
760
+ relation=relation.id,
761
+ )
762
+ if not relation.single_per_from:
763
+ self.add(
764
+ IssueCode.SCHEMA_FOREIGN_KEY_INVALID,
765
+ (*base, "cardinality"),
766
+ "foreign key Relations allow one Link per from Record (many_to_one or one_to_one)",
767
+ relation=relation.id,
768
+ )
769
+ from_kind = self.schema.kinds.get(relation.from_kind)
770
+ if relation.foreign_key_attribute is None:
771
+ self.add(
772
+ IssueCode.SCHEMA_FOREIGN_KEY_INVALID,
773
+ fk_path,
774
+ "foreign key attribute is missing",
775
+ relation=relation.id,
776
+ )
777
+ elif from_kind is not None:
778
+ attribute = from_kind.attributes.get(relation.foreign_key_attribute)
779
+ if attribute is None:
780
+ self.add(
781
+ IssueCode.SCHEMA_FOREIGN_KEY_INVALID,
782
+ fk_path,
783
+ f"{relation.foreign_key_attribute} is not an attribute of the from Kind",
784
+ relation=relation.id,
785
+ )
786
+ elif attribute.type != TypeSpec.ref(relation.to_kind):
787
+ self.add(
788
+ IssueCode.SCHEMA_FOREIGN_KEY_INVALID,
789
+ fk_path,
790
+ f"foreign key attribute {attribute.api_name!r} must be a ref to the to Kind",
791
+ relation=relation.id,
792
+ attribute=attribute.id,
793
+ )
794
+
795
+ # Traits ------------------------------------------------------------------------
796
+
797
+ def check_traits(self) -> None:
798
+ for trait in self.schema.traits.values():
799
+ for attribute in trait.attributes.values():
800
+ self.check_type_references(
801
+ attribute.type, ("traits", trait.id, "attributes", attribute.id, "type")
802
+ )
803
+
804
+ # Operations ----------------------------------------------------------------------
805
+
806
+ def check_operation(self, operation: Operation) -> None:
807
+ base: tuple[PathPart, ...] = ("operations", operation.id)
808
+ if operation.uses_expressions and not self.rules.allow_expressions:
809
+ self.add(
810
+ IssueCode.SCHEMA_EXPRESSIONS_DISABLED,
811
+ base,
812
+ f"Operation {operation.api_name!r} uses expressions, which are not enabled",
813
+ )
814
+ for index, parameter in enumerate(operation.parameters):
815
+ self.check_type_references(parameter.type, (*base, "parameters", index, "type"))
816
+ for index, step in enumerate(operation.steps):
817
+ path = (*base, "steps", index)
818
+ if isinstance(step, CreateLinkStep | DeleteLinkStep):
819
+ relation = self.schema.relations.get(step.relation)
820
+ if relation is None:
821
+ self.add(
822
+ IssueCode.SCHEMA_OPERATION_REFERENCE_UNKNOWN,
823
+ (*path, "relation"),
824
+ f"unknown Relation {step.relation}",
825
+ )
826
+ continue
827
+ if relation.backing is RelationBacking.FOREIGN_KEY:
828
+ self.add(
829
+ IssueCode.SCHEMA_OPERATION_INVALID,
830
+ (*path, "relation"),
831
+ f"Links of foreign key Relation {relation.api_name!r} are derived "
832
+ "and cannot be written",
833
+ relation=relation.id,
834
+ )
835
+ if isinstance(step, CreateLinkStep):
836
+ self.check_step_values(
837
+ step.values, relation.attributes, path, f"Relation {relation.api_name!r}"
838
+ )
839
+ continue
840
+ kind = self.schema.kinds.get(step.kind)
841
+ if kind is None:
842
+ self.add(
843
+ IssueCode.SCHEMA_OPERATION_REFERENCE_UNKNOWN,
844
+ (*path, "kind"),
845
+ f"unknown Kind {step.kind}",
846
+ )
847
+ continue
848
+ if isinstance(step, CreateRecordStep | UpdateRecordStep):
849
+ self.check_step_values(
850
+ step.values, kind.attributes, path, f"Kind {kind.api_name!r}"
851
+ )
852
+
853
+ def check_step_values(
854
+ self,
855
+ values: dict[str, str],
856
+ attributes: dict[str, Attribute],
857
+ path: tuple[PathPart, ...],
858
+ owner: str,
859
+ ) -> None:
860
+ for attribute_id in values:
861
+ attribute = attributes.get(attribute_id)
862
+ if attribute is None:
863
+ self.add(
864
+ IssueCode.SCHEMA_OPERATION_REFERENCE_UNKNOWN,
865
+ (*path, "values", attribute_id),
866
+ f"{attribute_id} is not an attribute of {owner}",
867
+ attribute=attribute_id,
868
+ )
869
+ elif attribute.is_derived:
870
+ self.add(
871
+ IssueCode.SCHEMA_OPERATION_INVALID,
872
+ (*path, "values", attribute_id),
873
+ f"derived attribute {attribute.api_name!r} cannot be written",
874
+ attribute=attribute_id,
875
+ )
876
+
877
+ # Derivations ---------------------------------------------------------------------
878
+
879
+ def check_derivation(self, derivation: Derivation) -> None:
880
+ base: tuple[PathPart, ...] = ("derivations", derivation.id, "definition")
881
+ definition = derivation.definition
882
+ uses_expression = isinstance(definition, ExpressionDerivation) or (
883
+ isinstance(definition, AggregateDerivation) and definition.filter is not None
884
+ )
885
+ if uses_expression and not self.rules.allow_expressions:
886
+ self.add(
887
+ IssueCode.SCHEMA_EXPRESSIONS_DISABLED,
888
+ base,
889
+ f"Derivation {derivation.api_name!r} uses expressions, which are not enabled",
890
+ )
891
+ if isinstance(definition, ExpressionDerivation | AggregateDerivation):
892
+ kind = self.schema.kinds.get(definition.kind)
893
+ if kind is None:
894
+ self.add(
895
+ IssueCode.SCHEMA_DERIVATION_REFERENCE_UNKNOWN,
896
+ (*base, "kind"),
897
+ f"unknown Kind {definition.kind}",
898
+ )
899
+ return
900
+ target = kind.attributes.get(definition.attribute)
901
+ if target is None:
902
+ self.add(
903
+ IssueCode.SCHEMA_DERIVATION_REFERENCE_UNKNOWN,
904
+ (*base, "attribute"),
905
+ f"{definition.attribute} is not an attribute of Kind {kind.api_name!r}",
906
+ kind=kind.id,
907
+ )
908
+ elif target.derivation_id != derivation.id:
909
+ self.add(
910
+ IssueCode.SCHEMA_DERIVATION_INVALID,
911
+ (*base, "attribute"),
912
+ f"attribute {target.api_name!r} does not declare this Derivation",
913
+ kind=kind.id,
914
+ attribute=target.id,
915
+ )
916
+ if isinstance(definition, AggregateDerivation):
917
+ self.check_aggregate(definition, kind, base)
918
+ return
919
+ previous_kind: str | None = None
920
+ for index, step in enumerate(definition.steps):
921
+ relation = self.schema.relations.get(step.relation)
922
+ if relation is None:
923
+ self.add(
924
+ IssueCode.SCHEMA_DERIVATION_REFERENCE_UNKNOWN,
925
+ (*base, "steps", index, "relation"),
926
+ f"unknown Relation {step.relation}",
927
+ )
928
+ previous_kind = None
929
+ continue
930
+ if previous_kind is not None and relation.endpoint(step.direction) != previous_kind:
931
+ self.add(
932
+ IssueCode.SCHEMA_DERIVATION_INVALID,
933
+ (*base, "steps", index),
934
+ f"Relation {relation.api_name!r} does not continue the path",
935
+ relation=relation.id,
936
+ )
937
+ previous_kind = relation.far_endpoint(step.direction)
938
+
939
+ def check_aggregate(
940
+ self, definition: AggregateDerivation, kind: Kind, base: tuple[PathPart, ...]
941
+ ) -> None:
942
+ relation = self.schema.link_relation(definition.relation)
943
+ if relation is None:
944
+ self.add(
945
+ IssueCode.SCHEMA_DERIVATION_REFERENCE_UNKNOWN,
946
+ (*base, "relation"),
947
+ f"unknown Relation {definition.relation}",
948
+ )
949
+ return
950
+ if relation.endpoint(definition.direction) != kind.id:
951
+ self.add(
952
+ IssueCode.SCHEMA_DERIVATION_INVALID,
953
+ (*base, "relation"),
954
+ f"Relation {relation.api_name!r} does not start at Kind {kind.api_name!r} "
955
+ f"in direction {definition.direction}",
956
+ relation=relation.id,
957
+ )
958
+ return
959
+ if definition.source_attribute is None:
960
+ return
961
+ far_kind = self.schema.kinds.get(relation.far_endpoint(definition.direction))
962
+ source = (
963
+ far_kind.attributes.get(definition.source_attribute) if far_kind is not None else None
964
+ )
965
+ if source is None:
966
+ self.add(
967
+ IssueCode.SCHEMA_DERIVATION_REFERENCE_UNKNOWN,
968
+ (*base, "source_attribute"),
969
+ f"{definition.source_attribute} is not an attribute of the Kind "
970
+ f"across {relation.api_name!r}",
971
+ )
972
+ elif definition.function in _NUMERIC_AGGREGATES and source.type.name not in _NUMERIC_TYPES:
973
+ self.add(
974
+ IssueCode.SCHEMA_DERIVATION_INVALID,
975
+ (*base, "function"),
976
+ f"{definition.function} needs a numeric source attribute, "
977
+ f"got {source.type.describe()}",
978
+ attribute=source.id,
979
+ )
980
+
981
+ def check_derivation_sensitivity(self) -> None:
982
+ """A derived value tells about what it reads: a derived Attribute carries every
983
+ sensitivity tag of the Attributes its Derivation may read, and never reads a vault
984
+ Attribute (which holds references)."""
985
+ from lorebase_core.expr.derivations import derivation_reads
986
+
987
+ for derivation in self.schema.derivations.values():
988
+ target = derivation.target
989
+ if target is None:
990
+ continue
991
+ kind = self.schema.kinds.get(target[0])
992
+ attribute = None if kind is None else kind.attributes.get(target[1])
993
+ if kind is None or attribute is None:
994
+ continue
995
+ needed: set[Sensitivity] = set()
996
+ vaulted: list[str] = []
997
+ reads = derivation_reads(self.schema, derivation, self.rules.expression_limits)
998
+ for kind_id, attribute_id in sorted(reads):
999
+ read_kind = self.schema.kinds.get(kind_id)
1000
+ read = None if read_kind is None else read_kind.attributes.get(attribute_id)
1001
+ if read is None or read.id == attribute.id:
1002
+ continue
1003
+ needed.update(read.sensitivity)
1004
+ if read.in_vault:
1005
+ vaulted.append(read.api_name)
1006
+ path: tuple[PathPart, ...] = ("kinds", kind.id, "attributes", attribute.id)
1007
+ missing = sorted(t.value for t in needed - set(attribute.sensitivity))
1008
+ if missing:
1009
+ self.add(
1010
+ IssueCode.SCHEMA_SENSITIVITY_INHERITED,
1011
+ (*path, "sensitivity"),
1012
+ f"derived attribute {attribute.api_name!r} reads sensitive values: it needs "
1013
+ f"the sensitivity tags {', '.join(missing)}",
1014
+ kind=kind.id,
1015
+ attribute=attribute.id,
1016
+ )
1017
+ if vaulted:
1018
+ self.add(
1019
+ IssueCode.SCHEMA_VAULT_INVALID,
1020
+ (*path, "derivation_id"),
1021
+ f"derived attribute {attribute.api_name!r} reads vault attributes "
1022
+ f"({', '.join(vaulted)}), whose stored values are references",
1023
+ kind=kind.id,
1024
+ attribute=attribute.id,
1025
+ )
1026
+
1027
+ def check_operation_sensitivity(self) -> None:
1028
+ """What an Operation computes from Records tells about them: a step value carries the
1029
+ tags of the Attributes its expression may read, and a precondition, whose outcome is
1030
+ reported to the caller, reads no sensitive Attribute. Neither reads a vault Attribute
1031
+ (Records hold references)."""
1032
+
1033
+ for operation in self.schema.operations.values():
1034
+ self.check_one_operation_sensitivity(operation)
1035
+
1036
+ def check_one_operation_sensitivity(self, operation: Operation) -> None:
1037
+ """`check_operation_sensitivity` for one Operation."""
1038
+ from lorebase_core.expr.sensitivity import operation_reads
1039
+
1040
+ limits = self.rules.expression_limits
1041
+ base: tuple[PathPart, ...] = ("operations", operation.id)
1042
+
1043
+ def quiet(
1044
+ expression: str, path: tuple[PathPart, ...], what: str, step: int | None = None
1045
+ ) -> None:
1046
+ """Expressions whose outcome reaches the caller (preconditions; the Records a step
1047
+ addresses, echoed back when missing) read no sensitive or vault Attribute."""
1048
+ read = operation_reads(self.schema, operation, expression, limits, step=step)
1049
+ sensitive = sorted({a.api_name for a in read if a.is_sensitive or a.in_vault})
1050
+ if sensitive:
1051
+ self.add(
1052
+ IssueCode.SCHEMA_SENSITIVITY_INHERITED,
1053
+ path,
1054
+ f"{what} of Operation {operation.api_name!r} reads sensitive attributes "
1055
+ f"({', '.join(sensitive)}): its outcome would tell about their values",
1056
+ )
1057
+
1058
+ for index, precondition in enumerate(operation.preconditions):
1059
+ quiet(
1060
+ precondition.expression,
1061
+ (*base, "preconditions", index, "expression"),
1062
+ "precondition",
1063
+ )
1064
+ for index, step in enumerate(operation.steps):
1065
+ if isinstance(step, UpdateRecordStep | DeleteRecordStep):
1066
+ quiet(step.target, (*base, "steps", index, "target"), "a step target", index)
1067
+ if isinstance(step, CreateLinkStep | DeleteLinkStep):
1068
+ quiet(step.from_, (*base, "steps", index, "from"), "a step source", index)
1069
+ quiet(step.to, (*base, "steps", index, "to"), "a step target", index)
1070
+ for index, step in enumerate(operation.steps):
1071
+ owner = (
1072
+ self.schema.kinds.get(step.kind)
1073
+ if isinstance(step, CreateRecordStep | UpdateRecordStep)
1074
+ else self.schema.relations.get(step.relation)
1075
+ if isinstance(step, CreateLinkStep)
1076
+ else None
1077
+ )
1078
+ if owner is None or isinstance(step, DeleteRecordStep | DeleteLinkStep):
1079
+ continue
1080
+ for attribute_id, expression in step.values.items():
1081
+ target = owner.attributes.get(attribute_id)
1082
+ if target is None:
1083
+ continue
1084
+ read = operation_reads(self.schema, operation, expression, limits, step=index)
1085
+ path = (*base, "steps", index, "values", attribute_id)
1086
+ needed = {t for a in read for t in a.sensitivity} - set(target.sensitivity)
1087
+ if needed:
1088
+ self.add(
1089
+ IssueCode.SCHEMA_SENSITIVITY_INHERITED,
1090
+ path,
1091
+ f"step value of {target.api_name!r} reads sensitive attributes: "
1092
+ f"the attribute needs the tags {', '.join(sorted(needed))}",
1093
+ attribute=target.id,
1094
+ )
1095
+ vaulted = sorted({a.api_name for a in read if a.in_vault})
1096
+ if vaulted:
1097
+ self.add(
1098
+ IssueCode.SCHEMA_VAULT_INVALID,
1099
+ path,
1100
+ f"step value of {target.api_name!r} reads vault attributes "
1101
+ f"({', '.join(vaulted)}), whose stored values are references",
1102
+ attribute=target.id,
1103
+ )
1104
+
1105
+ def check_derivation_cycles(self) -> None:
1106
+ """Derivations must not depend on themselves through derived Attributes: an aggregate
1107
+ over a derived ``source_attribute``, or (with expressions enabled) an expression or
1108
+ filter that reads a derived Attribute of its Record or of a Record one hop away."""
1109
+ edges: dict[str, set[str]] = defaultdict(set)
1110
+ owner: dict[str, str] = {}
1111
+ inputs_of = None
1112
+ if self.rules.allow_expressions:
1113
+ from lorebase_core.expr.derivations import derivation_inputs
1114
+
1115
+ inputs_of = derivation_inputs
1116
+ for derivation in self.schema.derivations.values():
1117
+ definition = derivation.definition
1118
+ if not isinstance(definition, AggregateDerivation | ExpressionDerivation):
1119
+ continue
1120
+ owner[definition.attribute] = derivation.id
1121
+ if inputs_of is not None:
1122
+ inputs = inputs_of(self.schema, derivation, self.rules.expression_limits)
1123
+ edges[definition.attribute].update(inputs)
1124
+ continue
1125
+ if (
1126
+ not isinstance(definition, AggregateDerivation)
1127
+ or definition.source_attribute is None
1128
+ ):
1129
+ continue
1130
+ relation = self.schema.link_relation(definition.relation)
1131
+ if relation is None:
1132
+ continue
1133
+ far_kind = self.schema.kinds.get(relation.far_endpoint(definition.direction))
1134
+ source = (
1135
+ far_kind.attributes.get(definition.source_attribute)
1136
+ if far_kind is not None
1137
+ else None
1138
+ )
1139
+ if source is not None and source.is_derived:
1140
+ edges[definition.attribute].add(source.id)
1141
+ visiting: set[str] = set()
1142
+ done: set[str] = set()
1143
+ reported: set[frozenset[str]] = set()
1144
+
1145
+ def visit(node: str, stack: list[str]) -> None:
1146
+ visiting.add(node)
1147
+ stack.append(node)
1148
+ for successor in sorted(edges.get(node, ())):
1149
+ if successor in visiting:
1150
+ cycle = stack[stack.index(successor) :]
1151
+ signature = frozenset(cycle)
1152
+ if signature not in reported:
1153
+ reported.add(signature)
1154
+ derivation_id = owner.get(cycle[0], "")
1155
+ self.add(
1156
+ IssueCode.SCHEMA_DERIVATION_CYCLE,
1157
+ ("derivations", derivation_id),
1158
+ "Derivations depend on each other in a cycle: "
1159
+ f"{' -> '.join([*cycle, successor])}",
1160
+ attribute=cycle[0],
1161
+ )
1162
+ elif successor not in done:
1163
+ visit(successor, stack)
1164
+ stack.pop()
1165
+ visiting.discard(node)
1166
+ done.add(node)
1167
+
1168
+ for node in sorted(edges):
1169
+ if node not in done:
1170
+ visit(node, [])
1171
+
1172
+ # Sources -------------------------------------------------------------------------
1173
+
1174
+ def check_source(self, source: Source) -> None:
1175
+ base: tuple[PathPart, ...] = ("sources", source.id)
1176
+ if not source.connection.startswith(SECRET_REFERENCE_PREFIX):
1177
+ self.add(
1178
+ IssueCode.SCHEMA_SOURCE_INVALID,
1179
+ (*base, "connection"),
1180
+ f"connection must be a secret reference ({SECRET_REFERENCE_PREFIX}...), "
1181
+ "never a literal",
1182
+ )
1183
+ if source.type in _QUERY_SOURCE_TYPES and not (source.query or "").strip():
1184
+ self.add(
1185
+ IssueCode.SCHEMA_SOURCE_INVALID,
1186
+ (*base, "query"),
1187
+ f"{source.type.value} Sources need a query",
1188
+ )
1189
+ for name, types in _SOURCE_TYPE_FIELDS.items():
1190
+ if getattr(source, name) is not None and source.type not in types:
1191
+ self.add(
1192
+ IssueCode.SCHEMA_SOURCE_INVALID,
1193
+ (*base, name),
1194
+ f"{name} applies to {', '.join(t.value for t in types)} Sources only",
1195
+ )
1196
+ if source.sync_mode is SyncMode.INCREMENTAL and source.watermark_column is None:
1197
+ self.add(
1198
+ IssueCode.SCHEMA_SOURCE_INVALID,
1199
+ (*base, "watermark_column"),
1200
+ "incremental Sources need a watermark column",
1201
+ )
1202
+ kind = self.schema.kinds.get(source.target_kind)
1203
+ if kind is None:
1204
+ self.add(
1205
+ IssueCode.SCHEMA_SOURCE_REFERENCE_UNKNOWN,
1206
+ (*base, "target_kind"),
1207
+ f"unknown Kind {source.target_kind}",
1208
+ )
1209
+ return
1210
+ if not kind.has_natural_key:
1211
+ self.add(
1212
+ IssueCode.SCHEMA_SOURCE_INVALID,
1213
+ (*base, "target_kind"),
1214
+ f"Kind {kind.api_name!r} has no primary key for the Source to map",
1215
+ kind=kind.id,
1216
+ )
1217
+ elif len(source.key_mapping) != len(kind.primary_key):
1218
+ self.add(
1219
+ IssueCode.SCHEMA_SOURCE_INVALID,
1220
+ (*base, "key_mapping"),
1221
+ f"key mapping has {len(source.key_mapping)} columns, "
1222
+ f"the primary key has {len(kind.primary_key)}",
1223
+ kind=kind.id,
1224
+ )
1225
+ for attribute_id in source.attribute_mapping:
1226
+ attribute = kind.attributes.get(attribute_id)
1227
+ if attribute is None:
1228
+ self.add(
1229
+ IssueCode.SCHEMA_SOURCE_REFERENCE_UNKNOWN,
1230
+ (*base, "attribute_mapping", attribute_id),
1231
+ f"{attribute_id} is not an attribute of Kind {kind.api_name!r}",
1232
+ kind=kind.id,
1233
+ attribute=attribute_id,
1234
+ )
1235
+ elif attribute.is_derived:
1236
+ self.add(
1237
+ IssueCode.SCHEMA_SOURCE_INVALID,
1238
+ (*base, "attribute_mapping", attribute_id),
1239
+ f"derived attribute {attribute.api_name!r} cannot be mapped",
1240
+ kind=kind.id,
1241
+ attribute=attribute_id,
1242
+ )
1243
+ for index, mapping in enumerate(source.relation_mapping):
1244
+ relation = self.schema.relations.get(mapping.relation)
1245
+ path = (*base, "relation_mapping", index, "relation")
1246
+ if relation is None:
1247
+ self.add(
1248
+ IssueCode.SCHEMA_SOURCE_REFERENCE_UNKNOWN,
1249
+ path,
1250
+ f"unknown Relation {mapping.relation}",
1251
+ )
1252
+ elif relation.from_kind != kind.id:
1253
+ self.add(
1254
+ IssueCode.SCHEMA_SOURCE_INVALID,
1255
+ path,
1256
+ f"Relation {relation.api_name!r} does not start at Kind {kind.api_name!r}",
1257
+ relation=relation.id,
1258
+ )
1259
+
1260
+ # Entry point ---------------------------------------------------------------------
1261
+
1262
+ def run(self) -> list[Issue]:
1263
+ self.check_names()
1264
+ self.check_attribute_ids()
1265
+ for kind in self.schema.kinds.values():
1266
+ self.check_kind(kind)
1267
+ for relation in self.schema.relations.values():
1268
+ self.check_relation(relation)
1269
+ self.check_traits()
1270
+ for operation in self.schema.operations.values():
1271
+ self.check_operation(operation)
1272
+ for derivation in self.schema.derivations.values():
1273
+ self.check_derivation(derivation)
1274
+ self.check_derivation_cycles()
1275
+ self.check_derivation_sensitivity()
1276
+ self.check_operation_sensitivity()
1277
+ for source in self.schema.sources.values():
1278
+ self.check_source(source)
1279
+ if self.rules.allow_expressions:
1280
+ self.check_expressions()
1281
+ return self.issues
1282
+
1283
+ def check_expressions(self) -> None:
1284
+ """Parse and type check every expression. Imported here because the expression
1285
+ engine itself imports this package."""
1286
+ from lorebase_core.expr.validation import schema_expression_issues
1287
+
1288
+ self.issues.extend(schema_expression_issues(self.schema, self.rules.expression_limits))
1289
+
1290
+
1291
+ def validate_schema(schema: Schema, rules: SchemaRules | None = None) -> list[Issue]:
1292
+ """Return every static problem of ``schema``; an empty list means the schema is valid."""
1293
+ return _Checker(schema, rules or SchemaRules()).run()
1294
+
1295
+
1296
+ def operation_sensitivity_issues(
1297
+ schema: Schema, operation: Operation, rules: SchemaRules | None = None
1298
+ ) -> list[Issue]:
1299
+ """The sensitivity rules of one Operation (``schema.sensitivity_inherited``,
1300
+ ``schema.vault_invalid``) against ``schema``, for instance with the default branch's tags
1301
+ added (``lorebase_core.expr.sensitivity.with_tags``)."""
1302
+ checker = _Checker(schema, rules or SchemaRules())
1303
+ checker.check_one_operation_sensitivity(operation)
1304
+ return checker.issues