validibot-shared 0.8.0__tar.gz → 0.9.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {validibot_shared-0.8.0 → validibot_shared-0.9.1}/PKG-INFO +3 -3
- {validibot_shared-0.8.0 → validibot_shared-0.9.1}/pyproject.toml +4 -2
- validibot_shared-0.9.1/tests/test_shacl_envelopes.py +267 -0
- {validibot_shared-0.8.0 → validibot_shared-0.9.1}/validibot_shared/__init__.py +12 -0
- validibot_shared-0.9.1/validibot_shared/shacl/__init__.py +35 -0
- validibot_shared-0.9.1/validibot_shared/shacl/envelopes.py +344 -0
- {validibot_shared-0.8.0 → validibot_shared-0.9.1}/validibot_shared/validations/envelopes.py +11 -0
- {validibot_shared-0.8.0 → validibot_shared-0.9.1}/.gitignore +0 -0
- {validibot_shared-0.8.0 → validibot_shared-0.9.1}/LICENSE +0 -0
- {validibot_shared-0.8.0 → validibot_shared-0.9.1}/NOTICE +0 -0
- {validibot_shared-0.8.0 → validibot_shared-0.9.1}/README.md +0 -0
- {validibot_shared-0.8.0 → validibot_shared-0.9.1}/tests/__init__.py +0 -0
- {validibot_shared-0.8.0 → validibot_shared-0.9.1}/tests/test_energyplus_envelopes.py +0 -0
- {validibot_shared-0.8.0 → validibot_shared-0.9.1}/tests/test_energyplus_models.py +0 -0
- {validibot_shared-0.8.0 → validibot_shared-0.9.1}/tests/test_evidence_manifest.py +0 -0
- {validibot_shared-0.8.0 → validibot_shared-0.9.1}/tests/test_fmu_envelopes.py +0 -0
- {validibot_shared-0.8.0 → validibot_shared-0.9.1}/tests/test_fmu_models.py +0 -0
- {validibot_shared-0.8.0 → validibot_shared-0.9.1}/tests/test_package_init.py +0 -0
- {validibot_shared-0.8.0 → validibot_shared-0.9.1}/tests/test_validations_envelopes.py +0 -0
- {validibot_shared-0.8.0 → validibot_shared-0.9.1}/validibot_shared/energyplus/__init__.py +0 -0
- {validibot_shared-0.8.0 → validibot_shared-0.9.1}/validibot_shared/energyplus/envelopes.py +0 -0
- {validibot_shared-0.8.0 → validibot_shared-0.9.1}/validibot_shared/energyplus/models.py +0 -0
- {validibot_shared-0.8.0 → validibot_shared-0.9.1}/validibot_shared/evidence/__init__.py +0 -0
- {validibot_shared-0.8.0 → validibot_shared-0.9.1}/validibot_shared/evidence/manifest.py +0 -0
- {validibot_shared-0.8.0 → validibot_shared-0.9.1}/validibot_shared/fmu/__init__.py +0 -0
- {validibot_shared-0.8.0 → validibot_shared-0.9.1}/validibot_shared/fmu/envelopes.py +0 -0
- {validibot_shared-0.8.0 → validibot_shared-0.9.1}/validibot_shared/fmu/models.py +0 -0
- {validibot_shared-0.8.0 → validibot_shared-0.9.1}/validibot_shared/py.typed +0 -0
- {validibot_shared-0.8.0 → validibot_shared-0.9.1}/validibot_shared/validations/__init__.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: validibot-shared
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.9.1
|
|
4
4
|
Summary: Shared library for data interchange between Validibot and validator containers
|
|
5
5
|
Project-URL: Homepage, https://validibot.com
|
|
6
6
|
Project-URL: Documentation, https://docs.validibot.com
|
|
@@ -11,7 +11,7 @@ Maintainer-email: Daniel McQuillen <daniel@validibot.com>
|
|
|
11
11
|
License: MIT
|
|
12
12
|
License-File: LICENSE
|
|
13
13
|
License-File: NOTICE
|
|
14
|
-
Keywords: data-interchange,energyplus,fmu,simulation,validation,validibot
|
|
14
|
+
Keywords: data-interchange,energyplus,fmu,rdf,shacl,simulation,validation,validibot
|
|
15
15
|
Classifier: Development Status :: 4 - Beta
|
|
16
16
|
Classifier: Intended Audience :: Developers
|
|
17
17
|
Classifier: License :: OSI Approved :: MIT License
|
|
@@ -27,7 +27,7 @@ Requires-Python: >=3.10
|
|
|
27
27
|
Requires-Dist: pydantic<3.0,>=2.13
|
|
28
28
|
Provides-Extra: dev
|
|
29
29
|
Requires-Dist: pytest==9.0.3; extra == 'dev'
|
|
30
|
-
Requires-Dist: ruff==0.15.
|
|
30
|
+
Requires-Dist: ruff==0.15.15; extra == 'dev'
|
|
31
31
|
Description-Content-Type: text/markdown
|
|
32
32
|
|
|
33
33
|
<div align="center">
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "validibot-shared"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.9.1"
|
|
8
8
|
description = "Shared library for data interchange between Validibot and validator containers"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = { text = "MIT" }
|
|
@@ -16,6 +16,8 @@ keywords = [
|
|
|
16
16
|
"validation",
|
|
17
17
|
"energyplus",
|
|
18
18
|
"fmu",
|
|
19
|
+
"shacl",
|
|
20
|
+
"rdf",
|
|
19
21
|
"simulation",
|
|
20
22
|
"data-interchange",
|
|
21
23
|
]
|
|
@@ -35,7 +37,7 @@ classifiers = [
|
|
|
35
37
|
dependencies = ["pydantic>=2.13,<3.0"]
|
|
36
38
|
|
|
37
39
|
[project.optional-dependencies]
|
|
38
|
-
dev = ["pytest==9.0.3", "ruff==0.15.
|
|
40
|
+
dev = ["pytest==9.0.3", "ruff==0.15.15"]
|
|
39
41
|
|
|
40
42
|
[project.urls]
|
|
41
43
|
Homepage = "https://validibot.com"
|
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
"""Tests for the SHACL Advanced validator envelopes.
|
|
2
|
+
|
|
3
|
+
This suite covers the contract that lets Django hand SHACL validation off to an
|
|
4
|
+
isolated container backend. The container parses untrusted RDF and runs
|
|
5
|
+
author-supplied SPARQL, so the envelope boundary is security-critical: anything
|
|
6
|
+
the container needs must travel *in* the envelope (no database, no settings, no
|
|
7
|
+
secrets reachable from the container), and anything it produces must round-trip
|
|
8
|
+
back to Django through ``SHACLOutputs``.
|
|
9
|
+
|
|
10
|
+
We therefore assert three things:
|
|
11
|
+
|
|
12
|
+
1. The typed envelopes accept well-formed payloads and round-trip through JSON
|
|
13
|
+
(Django serializes the input; the container serializes the output).
|
|
14
|
+
2. ``extra="forbid"`` is honoured so a drifting producer/consumer fails loudly
|
|
15
|
+
rather than silently dropping fields across the trust boundary.
|
|
16
|
+
3. The ``SHACLOutputs`` signal fields stay aligned with what Django's
|
|
17
|
+
``extract_output_signals`` expects (the ``o.*`` catalog keys).
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
import pytest
|
|
21
|
+
from pydantic import ValidationError
|
|
22
|
+
|
|
23
|
+
from validibot_shared.shacl.envelopes import (
|
|
24
|
+
SHACL_RESULT_REPORT_ONLY,
|
|
25
|
+
SHACLInputEnvelope,
|
|
26
|
+
SHACLInputs,
|
|
27
|
+
SHACLOutputEnvelope,
|
|
28
|
+
SHACLOutputs,
|
|
29
|
+
SHACLSparqlAssertionSpec,
|
|
30
|
+
build_shacl_input_envelope,
|
|
31
|
+
mime_type_for_rdf_format,
|
|
32
|
+
)
|
|
33
|
+
from validibot_shared.validations.envelopes import (
|
|
34
|
+
ExecutionContext,
|
|
35
|
+
SupportedMimeType,
|
|
36
|
+
ValidationStatus,
|
|
37
|
+
ValidatorType,
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
# The o.* signal keys Django's SHACL catalog declares. If SHACLOutputs ever
|
|
41
|
+
# drops one of these, the Django extractor would silently lose a CEL signal —
|
|
42
|
+
# this list is the canary.
|
|
43
|
+
EXPECTED_SIGNAL_KEYS = {
|
|
44
|
+
"parse_ok",
|
|
45
|
+
"parse_serialization",
|
|
46
|
+
"triple_count",
|
|
47
|
+
"namespaces_present",
|
|
48
|
+
"has_s223_namespace",
|
|
49
|
+
"has_g36_namespace",
|
|
50
|
+
"has_brick_namespace",
|
|
51
|
+
"shacl_violation_count",
|
|
52
|
+
"shacl_warning_count",
|
|
53
|
+
"shacl_info_count",
|
|
54
|
+
"shacl_total_count",
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _base_kwargs():
|
|
59
|
+
"""Minimal valid envelope metadata shared by the input-envelope tests."""
|
|
60
|
+
return {
|
|
61
|
+
"run_id": "run-123",
|
|
62
|
+
"validator": {"id": "val-1", "type": ValidatorType.SHACL, "version": "1"},
|
|
63
|
+
"org": {"id": "org-1", "name": "ValidiBot"},
|
|
64
|
+
"workflow": {"id": "wf-1", "step_id": "step-1", "step_name": "SHACL"},
|
|
65
|
+
"context": ExecutionContext(
|
|
66
|
+
callback_url="https://example.com/cb",
|
|
67
|
+
execution_bundle_uri="gs://bucket/run-123/",
|
|
68
|
+
),
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
# ── Input envelope ──────────────────────────────────────────────────────────
|
|
73
|
+
# The input envelope is what Django writes to storage before triggering the
|
|
74
|
+
# container. It must carry the merged shapes, resolved settings, and the
|
|
75
|
+
# author's SPARQL-ASK assertions, because the container has no DB to read them.
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def test_shacl_input_envelope_requires_typed_inputs():
|
|
79
|
+
"""The envelope must coerce a raw dict into a typed ``SHACLInputs``.
|
|
80
|
+
|
|
81
|
+
Django builds the inputs as a dict in some code paths; the typed override on
|
|
82
|
+
``SHACLInputEnvelope.inputs`` is what guarantees the container receives a
|
|
83
|
+
validated structure rather than arbitrary JSON.
|
|
84
|
+
"""
|
|
85
|
+
envelope = SHACLInputEnvelope(
|
|
86
|
+
**_base_kwargs(),
|
|
87
|
+
input_files=[],
|
|
88
|
+
inputs={
|
|
89
|
+
"shapes_text": "@prefix sh: <http://www.w3.org/ns/shacl#> .",
|
|
90
|
+
"rdf_format": "turtle",
|
|
91
|
+
"inference_mode": "rdfs",
|
|
92
|
+
"sparql_ask_assertions": [
|
|
93
|
+
{"target_graph": "data", "query": "ASK { ?s ?p ?o }"},
|
|
94
|
+
],
|
|
95
|
+
},
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
assert envelope.inputs.shapes_text.startswith("@prefix sh:")
|
|
99
|
+
assert envelope.inputs.sparql_ask_assertions[0].target_graph == "data"
|
|
100
|
+
# advanced features default off — the deployment gate must be opt-in.
|
|
101
|
+
assert envelope.inputs.advanced_shacl is False
|
|
102
|
+
assert envelope.inputs.enable_advanced_features is False
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def test_shacl_inputs_rejects_nonpositive_limits():
|
|
106
|
+
"""Resource limits must be positive so a corrupt envelope can't disable caps.
|
|
107
|
+
|
|
108
|
+
A zero/negative triple cap would mean "no limit" if it slipped through — the
|
|
109
|
+
container relies on these bounds as a DoS backstop, so the schema refuses
|
|
110
|
+
them at the boundary.
|
|
111
|
+
"""
|
|
112
|
+
with pytest.raises(ValidationError):
|
|
113
|
+
SHACLInputs(shapes_text="x", max_data_triples=0)
|
|
114
|
+
with pytest.raises(ValidationError):
|
|
115
|
+
SHACLInputs(shapes_text="x", pyshacl_timeout_seconds=-1)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def test_shacl_input_envelope_forbids_unknown_fields():
|
|
119
|
+
"""Unknown top-level keys are rejected (``extra='forbid'``).
|
|
120
|
+
|
|
121
|
+
Silent field-dropping across the Django/container trust boundary would let a
|
|
122
|
+
contract mismatch pass unnoticed; we want a hard failure instead.
|
|
123
|
+
"""
|
|
124
|
+
with pytest.raises(ValidationError):
|
|
125
|
+
SHACLInputEnvelope(
|
|
126
|
+
**_base_kwargs(),
|
|
127
|
+
input_files=[],
|
|
128
|
+
inputs={"shapes_text": "x"},
|
|
129
|
+
unexpected="nope",
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def test_build_shacl_input_envelope_constructs_expected_payload():
|
|
134
|
+
"""The helper wires the submission file, role, and MIME type correctly.
|
|
135
|
+
|
|
136
|
+
The container reads ``input_files[0].uri`` to fetch the RDF graph, and the
|
|
137
|
+
MIME type is derived from the resolved rdflib format — this proves the helper
|
|
138
|
+
produces a fetchable, correctly-typed input file.
|
|
139
|
+
"""
|
|
140
|
+
inputs = SHACLInputs(
|
|
141
|
+
shapes_text="@prefix sh: <http://www.w3.org/ns/shacl#> .",
|
|
142
|
+
rdf_format="json-ld",
|
|
143
|
+
)
|
|
144
|
+
envelope = build_shacl_input_envelope(
|
|
145
|
+
run_id="run-1",
|
|
146
|
+
validator=type(
|
|
147
|
+
"Validator",
|
|
148
|
+
(),
|
|
149
|
+
{"id": 1, "validation_type": ValidatorType.SHACL, "version": "2"},
|
|
150
|
+
)(),
|
|
151
|
+
org_id="org-1",
|
|
152
|
+
org_name="ValidiBot",
|
|
153
|
+
workflow_id="wf-1",
|
|
154
|
+
step_id="step-1",
|
|
155
|
+
step_name="Validate graph",
|
|
156
|
+
submission_uri="gs://bucket/submission.jsonld",
|
|
157
|
+
inputs=inputs,
|
|
158
|
+
callback_url="https://example.com/callback",
|
|
159
|
+
execution_bundle_uri="gs://bucket/run-1/",
|
|
160
|
+
skip_callback=True,
|
|
161
|
+
)
|
|
162
|
+
|
|
163
|
+
assert envelope.input_files[0].uri == "gs://bucket/submission.jsonld"
|
|
164
|
+
assert envelope.input_files[0].role == "primary-model"
|
|
165
|
+
assert envelope.input_files[0].mime_type == SupportedMimeType.RDF_JSON_LD
|
|
166
|
+
assert envelope.validator.version == "2"
|
|
167
|
+
assert envelope.context.skip_callback is True
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def test_mime_type_for_rdf_format_defaults_to_turtle():
|
|
171
|
+
"""An unknown format falls back to Turtle, the most common SHACL encoding.
|
|
172
|
+
|
|
173
|
+
Defaulting (rather than raising) keeps a slightly-off format hint from
|
|
174
|
+
blocking a run; the container still parses using the explicit ``rdf_format``.
|
|
175
|
+
"""
|
|
176
|
+
assert mime_type_for_rdf_format("turtle") == SupportedMimeType.RDF_TURTLE
|
|
177
|
+
assert mime_type_for_rdf_format("xml") == SupportedMimeType.RDF_XML
|
|
178
|
+
assert mime_type_for_rdf_format("totally-unknown") == SupportedMimeType.RDF_TURTLE
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
# ── SPARQL-ASK spec ─────────────────────────────────────────────────────────
|
|
182
|
+
# SPARQL-ASK assertions touch the graph, so they execute in the container, not
|
|
183
|
+
# in Django. The spec is the serialized form of a RulesetAssertion row.
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def test_sparql_assertion_spec_defaults():
|
|
187
|
+
"""A spec needs only a query; everything else has a safe default.
|
|
188
|
+
|
|
189
|
+
Defaulting ``severity`` to ERROR and ``target_graph`` to the data graph
|
|
190
|
+
matches the engine's historical behaviour so existing assertions keep their
|
|
191
|
+
meaning after the move to the container.
|
|
192
|
+
"""
|
|
193
|
+
spec = SHACLSparqlAssertionSpec(query="ASK { ?s a ?type }")
|
|
194
|
+
assert spec.severity == "ERROR"
|
|
195
|
+
assert spec.target_graph == "data"
|
|
196
|
+
assert spec.assertion_id is None
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
# ── Output envelope ─────────────────────────────────────────────────────────
|
|
200
|
+
# The output envelope is what the container writes back. Its signal fields feed
|
|
201
|
+
# Django's CEL/Basic assertion evaluation; its assertion tallies are folded into
|
|
202
|
+
# the final AssertionStats.
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def test_shacl_outputs_signal_keys_cover_catalog():
|
|
206
|
+
"""Every catalog signal key must exist as a field on ``SHACLOutputs``.
|
|
207
|
+
|
|
208
|
+
Django's ``extract_output_signals`` reads these keys off the envelope; a
|
|
209
|
+
missing field here would surface as a null CEL signal at runtime with no
|
|
210
|
+
error. This test pins the alignment at the contract layer.
|
|
211
|
+
"""
|
|
212
|
+
field_names = set(SHACLOutputs.model_fields)
|
|
213
|
+
missing = EXPECTED_SIGNAL_KEYS - field_names
|
|
214
|
+
assert not missing, f"SHACLOutputs missing signal fields: {missing}"
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def test_shacl_output_envelope_round_trips():
|
|
218
|
+
"""A full output envelope round-trips through JSON unchanged.
|
|
219
|
+
|
|
220
|
+
Django deserializes ``output.json`` written by the container; this proves the
|
|
221
|
+
typed subclass reconstructs the ``SHACLOutputs`` (including the serialized
|
|
222
|
+
report and assertion counts) faithfully.
|
|
223
|
+
"""
|
|
224
|
+
envelope = SHACLOutputEnvelope(
|
|
225
|
+
run_id="run-123",
|
|
226
|
+
validator={"id": "val-1", "type": ValidatorType.SHACL, "version": "1"},
|
|
227
|
+
status=ValidationStatus.FAILED_VALIDATION,
|
|
228
|
+
timing={},
|
|
229
|
+
messages=[],
|
|
230
|
+
outputs=SHACLOutputs(
|
|
231
|
+
conforms=False,
|
|
232
|
+
parse_ok=True,
|
|
233
|
+
parse_serialization="turtle",
|
|
234
|
+
triple_count=42,
|
|
235
|
+
has_s223_namespace=True,
|
|
236
|
+
shacl_violation_count=2,
|
|
237
|
+
shacl_total_count=2,
|
|
238
|
+
results_graph_turtle="@prefix sh: <http://www.w3.org/ns/shacl#> .",
|
|
239
|
+
shacl_result_handling=SHACL_RESULT_REPORT_ONLY,
|
|
240
|
+
assertion_total=3,
|
|
241
|
+
assertion_failures=1,
|
|
242
|
+
execution_seconds=0.5,
|
|
243
|
+
),
|
|
244
|
+
)
|
|
245
|
+
|
|
246
|
+
restored = SHACLOutputEnvelope.model_validate_json(envelope.model_dump_json())
|
|
247
|
+
assert restored.outputs.conforms is False
|
|
248
|
+
assert restored.outputs.triple_count == 42
|
|
249
|
+
assert restored.outputs.assertion_failures == 1
|
|
250
|
+
assert restored.status == ValidationStatus.FAILED_VALIDATION
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def test_shacl_output_envelope_allows_none_outputs():
|
|
254
|
+
"""``outputs`` may be None for a runtime failure before SHACL produced a report.
|
|
255
|
+
|
|
256
|
+
The container sets a FAILED_RUNTIME status with ``outputs=None`` when, e.g.,
|
|
257
|
+
the submission fails to parse — Django must still be able to deserialize that
|
|
258
|
+
envelope to surface the failure.
|
|
259
|
+
"""
|
|
260
|
+
envelope = SHACLOutputEnvelope(
|
|
261
|
+
run_id="run-123",
|
|
262
|
+
validator={"id": "val-1", "type": ValidatorType.SHACL, "version": "1"},
|
|
263
|
+
status=ValidationStatus.FAILED_RUNTIME,
|
|
264
|
+
timing={},
|
|
265
|
+
outputs=None,
|
|
266
|
+
)
|
|
267
|
+
assert envelope.outputs is None
|
|
@@ -16,6 +16,13 @@ from validibot_shared.evidence import (
|
|
|
16
16
|
WorkflowContractSnapshot,
|
|
17
17
|
)
|
|
18
18
|
from validibot_shared.fmu.models import FMUProbeResult, FMUVariableMeta
|
|
19
|
+
from validibot_shared.shacl.envelopes import (
|
|
20
|
+
SHACLInputEnvelope,
|
|
21
|
+
SHACLInputs,
|
|
22
|
+
SHACLOutputEnvelope,
|
|
23
|
+
SHACLOutputs,
|
|
24
|
+
SHACLSparqlAssertionSpec,
|
|
25
|
+
)
|
|
19
26
|
|
|
20
27
|
__all__ = [
|
|
21
28
|
"EVIDENCE_SCHEMA_VERSION",
|
|
@@ -27,6 +34,11 @@ __all__ = [
|
|
|
27
34
|
"FMUVariableMeta",
|
|
28
35
|
"ManifestPayloadDigests",
|
|
29
36
|
"ManifestRetentionInfo",
|
|
37
|
+
"SHACLInputEnvelope",
|
|
38
|
+
"SHACLInputs",
|
|
39
|
+
"SHACLOutputEnvelope",
|
|
40
|
+
"SHACLOutputs",
|
|
41
|
+
"SHACLSparqlAssertionSpec",
|
|
30
42
|
"StepValidatorRecord",
|
|
31
43
|
"WorkflowContractSnapshot",
|
|
32
44
|
]
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"""SHACL Advanced validator envelopes and models."""
|
|
2
|
+
|
|
3
|
+
from validibot_shared.shacl.envelopes import (
|
|
4
|
+
SHACL_RESULT_FAIL_AFTER_ASSERTIONS,
|
|
5
|
+
SHACL_RESULT_FAIL_IMMEDIATELY,
|
|
6
|
+
SHACL_RESULT_REPORT_ONLY,
|
|
7
|
+
SPARQL_ASK_TARGET_DATA,
|
|
8
|
+
SPARQL_ASK_TARGET_RESULTS,
|
|
9
|
+
SPARQL_ASK_TARGET_UNION,
|
|
10
|
+
SHACLFinding,
|
|
11
|
+
SHACLInputEnvelope,
|
|
12
|
+
SHACLInputs,
|
|
13
|
+
SHACLOutputEnvelope,
|
|
14
|
+
SHACLOutputs,
|
|
15
|
+
SHACLSparqlAssertionSpec,
|
|
16
|
+
build_shacl_input_envelope,
|
|
17
|
+
mime_type_for_rdf_format,
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
__all__ = [
|
|
21
|
+
"SHACL_RESULT_FAIL_AFTER_ASSERTIONS",
|
|
22
|
+
"SHACL_RESULT_FAIL_IMMEDIATELY",
|
|
23
|
+
"SHACL_RESULT_REPORT_ONLY",
|
|
24
|
+
"SPARQL_ASK_TARGET_DATA",
|
|
25
|
+
"SPARQL_ASK_TARGET_RESULTS",
|
|
26
|
+
"SPARQL_ASK_TARGET_UNION",
|
|
27
|
+
"SHACLFinding",
|
|
28
|
+
"SHACLInputEnvelope",
|
|
29
|
+
"SHACLInputs",
|
|
30
|
+
"SHACLOutputEnvelope",
|
|
31
|
+
"SHACLOutputs",
|
|
32
|
+
"SHACLSparqlAssertionSpec",
|
|
33
|
+
"build_shacl_input_envelope",
|
|
34
|
+
"mime_type_for_rdf_format",
|
|
35
|
+
]
|
|
@@ -0,0 +1,344 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Pydantic envelopes for the SHACL Advanced validator backend.
|
|
3
|
+
|
|
4
|
+
The SHACL validator is an Advanced validator — it runs in an isolated
|
|
5
|
+
container/Cloud Run Job rather than inside the Django worker. That isolation is
|
|
6
|
+
the whole point: SHACL parses untrusted RDF and executes author-supplied SPARQL
|
|
7
|
+
(SHACL-AF constraints and SPARQL-ASK assertions), and we never want that running
|
|
8
|
+
next to the worker's database credentials, service-account identity, or network.
|
|
9
|
+
|
|
10
|
+
These schemas define the contract between Django and the SHACL container:
|
|
11
|
+
|
|
12
|
+
- **Input envelope**: the RDF submission (as an ``InputFileItem`` URI) plus the
|
|
13
|
+
merged shape/ontology text, the resolved engine settings, the resource limits,
|
|
14
|
+
and the author-defined SPARQL-ASK assertions. Django resolves all of this from
|
|
15
|
+
the database (rulesets, ``RulesetAssertion`` rows, settings) before dispatch —
|
|
16
|
+
the container receives only this serialized boundary.
|
|
17
|
+
- **Output envelope**: the SHACL findings (as ``ValidationMessage`` rows), the
|
|
18
|
+
``o.*`` signal dict (for CEL/Basic output assertions evaluated back in Django),
|
|
19
|
+
the serialized ``sh:ValidationReport`` for evidence download, and the
|
|
20
|
+
container-side SPARQL-ASK assertion counts (folded into the final assertion
|
|
21
|
+
totals by ``AdvancedValidator.post_execute_validate``).
|
|
22
|
+
|
|
23
|
+
See ADR-2026-05-18 for the SHACL engine design and the cross-repo plan for the
|
|
24
|
+
isolation rationale.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
from __future__ import annotations
|
|
28
|
+
|
|
29
|
+
from typing import Any
|
|
30
|
+
|
|
31
|
+
from pydantic import BaseModel, Field
|
|
32
|
+
|
|
33
|
+
from validibot_shared.validations.envelopes import (
|
|
34
|
+
ExecutionContext,
|
|
35
|
+
InputFileItem,
|
|
36
|
+
SupportedMimeType,
|
|
37
|
+
ValidationInputEnvelope,
|
|
38
|
+
ValidationOutputEnvelope,
|
|
39
|
+
ValidatorInfo,
|
|
40
|
+
ValidatorType,
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
# Result-handling modes. Mirrors
|
|
44
|
+
# ``validibot.validations.validators.shacl.constants`` on the Django side; the
|
|
45
|
+
# container applies the resolved mode to decide which findings block.
|
|
46
|
+
SHACL_RESULT_FAIL_IMMEDIATELY = "fail_immediately"
|
|
47
|
+
SHACL_RESULT_FAIL_AFTER_ASSERTIONS = "fail_after_assertions"
|
|
48
|
+
SHACL_RESULT_REPORT_ONLY = "report_only"
|
|
49
|
+
|
|
50
|
+
# SPARQL-ASK target graphs.
|
|
51
|
+
SPARQL_ASK_TARGET_DATA = "data"
|
|
52
|
+
SPARQL_ASK_TARGET_RESULTS = "results"
|
|
53
|
+
SPARQL_ASK_TARGET_UNION = "union"
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class SHACLSparqlAssertionSpec(BaseModel):
|
|
57
|
+
"""One author-defined SPARQL-ASK assertion to run inside the container.
|
|
58
|
+
|
|
59
|
+
Django rehydrates each ``RulesetAssertion`` row (``assertion_type=SHACL``)
|
|
60
|
+
into one of these specs and ships it in the input envelope. The container
|
|
61
|
+
re-scrubs ``query`` through the AST scrubber before executing it — the form
|
|
62
|
+
already scrubbed at save time, but the runtime re-check is belt-and-braces
|
|
63
|
+
against fixtures/imports that bypass the form.
|
|
64
|
+
"""
|
|
65
|
+
|
|
66
|
+
target_graph: str = Field(
|
|
67
|
+
default=SPARQL_ASK_TARGET_DATA,
|
|
68
|
+
description="Which graph to run the ASK against: data | results | union.",
|
|
69
|
+
)
|
|
70
|
+
query: str = Field(description="Raw SPARQL ASK query text.")
|
|
71
|
+
severity: str = Field(
|
|
72
|
+
default="ERROR",
|
|
73
|
+
description=(
|
|
74
|
+
"Validibot severity emitted on a false answer (ERROR/WARNING/INFO)."
|
|
75
|
+
),
|
|
76
|
+
)
|
|
77
|
+
description: str = Field(default="", description="Human-readable label.")
|
|
78
|
+
error_message_template: str = Field(
|
|
79
|
+
default="",
|
|
80
|
+
description="Message shown when the ASK returns false.",
|
|
81
|
+
)
|
|
82
|
+
success_message: str = Field(
|
|
83
|
+
default="",
|
|
84
|
+
description="Optional message recorded when the ASK returns true.",
|
|
85
|
+
)
|
|
86
|
+
assertion_id: int | None = Field(
|
|
87
|
+
default=None,
|
|
88
|
+
description="RulesetAssertion.pk, for finding attribution back in Django.",
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
model_config = {"extra": "forbid"}
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
class SHACLInputs(BaseModel):
|
|
95
|
+
"""SHACL engine configuration resolved by Django and run in the container.
|
|
96
|
+
|
|
97
|
+
Everything the container needs to validate without any database access. The
|
|
98
|
+
shape/ontology text is already merged (library default + step extras), the
|
|
99
|
+
RDF format is already resolved from the submission filename, and the resource
|
|
100
|
+
limits are already clamped to their hard caps Django-side — the container
|
|
101
|
+
treats these as authoritative but still hard-caps defensively.
|
|
102
|
+
"""
|
|
103
|
+
|
|
104
|
+
shapes_text: str = Field(description="Merged SHACL shapes graph (Turtle).")
|
|
105
|
+
ontology_text: str = Field(
|
|
106
|
+
default="",
|
|
107
|
+
description="Optional merged ontology graph (Turtle) for inference.",
|
|
108
|
+
)
|
|
109
|
+
rdf_format: str = Field(
|
|
110
|
+
default="turtle",
|
|
111
|
+
description=(
|
|
112
|
+
"Resolved rdflib format slug for the submission "
|
|
113
|
+
"(turtle | json-ld | xml | nt | nquads)."
|
|
114
|
+
),
|
|
115
|
+
)
|
|
116
|
+
inference_mode: str = Field(
|
|
117
|
+
default="rdfs",
|
|
118
|
+
description="pyshacl inference mode: none | rdfs | owlrl | both.",
|
|
119
|
+
)
|
|
120
|
+
advanced_shacl: bool = Field(
|
|
121
|
+
default=False,
|
|
122
|
+
description="Whether the author requested SHACL-AF / embedded SPARQL.",
|
|
123
|
+
)
|
|
124
|
+
enable_advanced_features: bool = Field(
|
|
125
|
+
default=False,
|
|
126
|
+
description=(
|
|
127
|
+
"Deployment gate forwarded from SHACL_ENABLE_ADVANCED_FEATURES. "
|
|
128
|
+
"Even when advanced_shacl is requested, embedded SPARQL execution is "
|
|
129
|
+
"only accepted when this is also true."
|
|
130
|
+
),
|
|
131
|
+
)
|
|
132
|
+
submission_format: str = Field(
|
|
133
|
+
default="auto",
|
|
134
|
+
description=(
|
|
135
|
+
"Original author-selected submission format (auto/turtle/...). "
|
|
136
|
+
"Retained for diagnostics; the container parses using rdf_format."
|
|
137
|
+
),
|
|
138
|
+
)
|
|
139
|
+
shacl_result_handling: str = Field(
|
|
140
|
+
default=SHACL_RESULT_FAIL_AFTER_ASSERTIONS,
|
|
141
|
+
description="fail_immediately | fail_after_assertions | report_only.",
|
|
142
|
+
)
|
|
143
|
+
bundled_standards: list[str] = Field(
|
|
144
|
+
default_factory=list,
|
|
145
|
+
description="Opted-in bundled standard identifiers (e.g. brick-1.4).",
|
|
146
|
+
)
|
|
147
|
+
sparql_ask_assertions: list[SHACLSparqlAssertionSpec] = Field(
|
|
148
|
+
default_factory=list,
|
|
149
|
+
description="Author-defined SPARQL-ASK assertions to evaluate post-SHACL.",
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
# ── Resource limits (already clamped Django-side; re-clamped in container) ──
|
|
153
|
+
max_data_triples: int = Field(default=100_000, gt=0)
|
|
154
|
+
max_shape_triples: int = Field(default=50_000, gt=0)
|
|
155
|
+
max_ontology_triples: int = Field(default=100_000, gt=0)
|
|
156
|
+
max_validation_depth: int = Field(default=25, gt=0)
|
|
157
|
+
# Mirror the producer-side defaults so a direct/shared-envelope consumer
|
|
158
|
+
# (or a test) that doesn't set these explicitly behaves like a real launch.
|
|
159
|
+
# Django (validations/.../shacl/launch.py) and the container backend
|
|
160
|
+
# (validibot-validator-backends shacl/engine.py) both default pySHACL to
|
|
161
|
+
# 300s (hard cap 1800s) and SPARQL-ASK to 10s; this contract must not lag.
|
|
162
|
+
pyshacl_timeout_seconds: int = Field(default=300, gt=0)
|
|
163
|
+
sparql_query_timeout_seconds: int = Field(default=10, gt=0)
|
|
164
|
+
|
|
165
|
+
model_config = {"extra": "forbid"}
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
class SHACLFinding(BaseModel):
|
|
169
|
+
"""One SHACL finding with the full detail Django
|
|
170
|
+
gneeds to rebuild a ``ValidationIssue``.
|
|
171
|
+
|
|
172
|
+
The generic ``ValidationMessage`` on the output envelope is lossy — it has no
|
|
173
|
+
``meta`` (SHACL focus node / source shape / constraint component) and no
|
|
174
|
+
``assertion_id`` (SPARQL-ASK attribution). SHACL carries those through this
|
|
175
|
+
richer per-finding model so Django's ``post_execute_validate`` can reconstruct
|
|
176
|
+
findings with the same fidelity as the old in-process path.
|
|
177
|
+
|
|
178
|
+
``severity`` is a plain string (ERROR / WARNING / INFO / SUCCESS) rather than
|
|
179
|
+
the shared ``Severity`` enum because SHACL emits SUCCESS rows for passing
|
|
180
|
+
SPARQL-ASK assertions with a configured success message, and ``Severity`` only
|
|
181
|
+
covers INFO/WARNING/ERROR. Django maps it back to its own severity enum.
|
|
182
|
+
"""
|
|
183
|
+
|
|
184
|
+
path: str = Field(default="")
|
|
185
|
+
message: str = Field(description="Human-readable finding text.")
|
|
186
|
+
severity: str = Field(description="ERROR | WARNING | INFO | SUCCESS.")
|
|
187
|
+
code: str = Field(default="")
|
|
188
|
+
meta: dict[str, Any] = Field(default_factory=dict)
|
|
189
|
+
assertion_id: int | None = Field(default=None)
|
|
190
|
+
|
|
191
|
+
model_config = {"extra": "forbid"}
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
class SHACLOutputs(BaseModel):
|
|
195
|
+
"""SHACL validation results and the ``o.*`` signal dict.
|
|
196
|
+
|
|
197
|
+
The signal fields (``parse_ok`` … ``shacl_total_count``) mirror the catalog
|
|
198
|
+
entries declared in the Django SHACL ``ValidatorConfig`` — Django's
|
|
199
|
+
``extract_output_signals`` pulls exactly those keys for CEL/Basic assertion
|
|
200
|
+
evaluation. The remaining fields carry evidence (the serialized report) and
|
|
201
|
+
the container-side SPARQL-ASK assertion counts.
|
|
202
|
+
"""
|
|
203
|
+
|
|
204
|
+
conforms: bool = Field(description="Whether the data graph conforms to the shapes.")
|
|
205
|
+
|
|
206
|
+
findings: list[SHACLFinding] = Field(
|
|
207
|
+
default_factory=list,
|
|
208
|
+
description=(
|
|
209
|
+
"Blocking + advisory findings (SHACL results, SPARQL-ASK outcomes, "
|
|
210
|
+
"bundle/parse/engine messages) with full meta for Django to rebuild "
|
|
211
|
+
"ValidationIssue rows. Already filtered per shacl_result_handling."
|
|
212
|
+
),
|
|
213
|
+
)
|
|
214
|
+
|
|
215
|
+
# ── o.* signals (must stay aligned with the SHACL ValidatorConfig catalog) ──
|
|
216
|
+
parse_ok: bool = Field(description="Whether the submitted RDF parsed.")
|
|
217
|
+
parse_serialization: str = Field(description="rdflib format used to parse.")
|
|
218
|
+
triple_count: int = Field(default=0, ge=0)
|
|
219
|
+
namespaces_present: list[str] = Field(default_factory=list)
|
|
220
|
+
has_s223_namespace: bool = Field(default=False)
|
|
221
|
+
has_g36_namespace: bool = Field(default=False)
|
|
222
|
+
has_brick_namespace: bool = Field(default=False)
|
|
223
|
+
shacl_violation_count: int = Field(default=0, ge=0)
|
|
224
|
+
shacl_warning_count: int = Field(default=0, ge=0)
|
|
225
|
+
shacl_info_count: int = Field(default=0, ge=0)
|
|
226
|
+
shacl_total_count: int = Field(default=0, ge=0)
|
|
227
|
+
|
|
228
|
+
# ── Evidence + run metadata ──
|
|
229
|
+
results_graph_turtle: str = Field(
|
|
230
|
+
default="",
|
|
231
|
+
description="Serialized sh:ValidationReport (Turtle) for evidence download.",
|
|
232
|
+
)
|
|
233
|
+
shacl_shapes_sha256: str = Field(default="")
|
|
234
|
+
shacl_ontology_sha256: str = Field(default="")
|
|
235
|
+
advanced_shacl_requested: bool = Field(default=False)
|
|
236
|
+
shacl_result_handling: str = Field(default="")
|
|
237
|
+
|
|
238
|
+
# ── Container-side SPARQL-ASK assertion tallies (folded into AssertionStats) ──
|
|
239
|
+
assertion_total: int = Field(default=0, ge=0)
|
|
240
|
+
assertion_failures: int = Field(default=0, ge=0)
|
|
241
|
+
|
|
242
|
+
execution_seconds: float = Field(default=0.0, ge=0)
|
|
243
|
+
pyshacl_version: str | None = Field(default=None)
|
|
244
|
+
|
|
245
|
+
model_config = {"extra": "forbid"}
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
class SHACLInputEnvelope(ValidationInputEnvelope):
|
|
249
|
+
"""Input envelope for SHACL validator containers."""
|
|
250
|
+
|
|
251
|
+
inputs: SHACLInputs
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
class SHACLOutputEnvelope(ValidationOutputEnvelope):
|
|
255
|
+
"""Output envelope from SHACL validator containers.
|
|
256
|
+
|
|
257
|
+
``outputs`` can be ``None`` for runtime-failure cases where the engine never
|
|
258
|
+
produced a report (e.g. the submission failed to parse before SHACL ran).
|
|
259
|
+
"""
|
|
260
|
+
|
|
261
|
+
outputs: SHACLOutputs | None = None
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
# Default file role/MIME for the submitted RDF graph. The role mirrors the
|
|
265
|
+
# "primary-model" convention used by EnergyPlus; the container reads
|
|
266
|
+
# input_files[0].uri regardless of role.
|
|
267
|
+
_RDF_FORMAT_TO_MIME: dict[str, SupportedMimeType] = {
|
|
268
|
+
"turtle": SupportedMimeType.RDF_TURTLE,
|
|
269
|
+
"n3": SupportedMimeType.RDF_TURTLE,
|
|
270
|
+
"json-ld": SupportedMimeType.RDF_JSON_LD,
|
|
271
|
+
"xml": SupportedMimeType.RDF_XML,
|
|
272
|
+
"nt": SupportedMimeType.RDF_N_TRIPLES,
|
|
273
|
+
"nquads": SupportedMimeType.RDF_N_QUADS,
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def mime_type_for_rdf_format(rdf_format: str) -> SupportedMimeType:
|
|
278
|
+
"""Map a resolved rdflib format slug to a SupportedMimeType (Turtle default)."""
|
|
279
|
+
return _RDF_FORMAT_TO_MIME.get(rdf_format, SupportedMimeType.RDF_TURTLE)
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def build_shacl_input_envelope(
|
|
283
|
+
*,
|
|
284
|
+
run_id: str,
|
|
285
|
+
validator,
|
|
286
|
+
org_id: str,
|
|
287
|
+
org_name: str,
|
|
288
|
+
workflow_id: str,
|
|
289
|
+
step_id: str,
|
|
290
|
+
step_name: str | None,
|
|
291
|
+
submission_uri: str,
|
|
292
|
+
inputs: SHACLInputs,
|
|
293
|
+
callback_url: str,
|
|
294
|
+
execution_bundle_uri: str,
|
|
295
|
+
callback_id: str | None = None,
|
|
296
|
+
skip_callback: bool = False,
|
|
297
|
+
) -> SHACLInputEnvelope:
|
|
298
|
+
"""Build a ``SHACLInputEnvelope`` from Django validation data.
|
|
299
|
+
|
|
300
|
+
Args:
|
|
301
|
+
run_id: ValidationRun ID.
|
|
302
|
+
validator: Validator-like object (id/validation_type/version attrs).
|
|
303
|
+
org_id / org_name: Organization identity.
|
|
304
|
+
workflow_id / step_id / step_name: Workflow context.
|
|
305
|
+
submission_uri: Storage URI for the RDF submission (gs:// or file://).
|
|
306
|
+
inputs: Fully-resolved ``SHACLInputs`` (shapes/ontology/settings/asks).
|
|
307
|
+
callback_url: URL the container POSTs to on completion.
|
|
308
|
+
execution_bundle_uri: Base URI for this run's bundle.
|
|
309
|
+
callback_id: Idempotency key echoed back in the callback.
|
|
310
|
+
skip_callback: True for synchronous (Docker) execution.
|
|
311
|
+
"""
|
|
312
|
+
input_files = [
|
|
313
|
+
InputFileItem(
|
|
314
|
+
name="submission.rdf",
|
|
315
|
+
mime_type=mime_type_for_rdf_format(inputs.rdf_format),
|
|
316
|
+
role="primary-model",
|
|
317
|
+
uri=submission_uri,
|
|
318
|
+
),
|
|
319
|
+
]
|
|
320
|
+
|
|
321
|
+
context = ExecutionContext(
|
|
322
|
+
callback_id=callback_id,
|
|
323
|
+
callback_url=callback_url,
|
|
324
|
+
execution_bundle_uri=execution_bundle_uri,
|
|
325
|
+
skip_callback=skip_callback,
|
|
326
|
+
)
|
|
327
|
+
|
|
328
|
+
return SHACLInputEnvelope(
|
|
329
|
+
run_id=run_id,
|
|
330
|
+
validator=ValidatorInfo(
|
|
331
|
+
id=str(validator.id),
|
|
332
|
+
type=ValidatorType(validator.validation_type),
|
|
333
|
+
version=str(getattr(validator, "version", "1")),
|
|
334
|
+
),
|
|
335
|
+
org={"id": org_id, "name": org_name},
|
|
336
|
+
workflow={
|
|
337
|
+
"id": workflow_id,
|
|
338
|
+
"step_id": step_id,
|
|
339
|
+
"step_name": step_name,
|
|
340
|
+
},
|
|
341
|
+
input_files=input_files,
|
|
342
|
+
inputs=inputs,
|
|
343
|
+
context=context,
|
|
344
|
+
)
|
|
@@ -143,6 +143,7 @@ class ValidatorType(str, Enum):
|
|
|
143
143
|
BASIC = "BASIC"
|
|
144
144
|
JSON_SCHEMA = "JSON_SCHEMA"
|
|
145
145
|
XML_SCHEMA = "XML_SCHEMA"
|
|
146
|
+
SHACL = "SHACL"
|
|
146
147
|
ENERGYPLUS = "ENERGYPLUS"
|
|
147
148
|
FMU = "FMU"
|
|
148
149
|
CUSTOM_VALIDATOR = "CUSTOM_VALIDATOR"
|
|
@@ -173,6 +174,16 @@ class SupportedMimeType(str, Enum):
|
|
|
173
174
|
# FMU files
|
|
174
175
|
FMU = "application/vnd.fmi.fmu" # Functional Mock-up Unit
|
|
175
176
|
|
|
177
|
+
# RDF serializations (SHACL validator). The concrete parse format is
|
|
178
|
+
# carried by SHACLInputs.submission_format; these MIME types let the RDF
|
|
179
|
+
# submission ride the typed InputFileItem.mime_type field like any other
|
|
180
|
+
# validator input.
|
|
181
|
+
RDF_TURTLE = "text/turtle"
|
|
182
|
+
RDF_XML = "application/rdf+xml"
|
|
183
|
+
RDF_JSON_LD = "application/ld+json"
|
|
184
|
+
RDF_N_TRIPLES = "application/n-triples"
|
|
185
|
+
RDF_N_QUADS = "application/n-quads"
|
|
186
|
+
|
|
176
187
|
|
|
177
188
|
class InputFileItem(BaseModel):
|
|
178
189
|
"""
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|