diffable-rdf 0.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- diffable_rdf-0.0.1/.gitignore +19 -0
- diffable_rdf-0.0.1/LICENSE +201 -0
- diffable_rdf-0.0.1/PKG-INFO +105 -0
- diffable_rdf-0.0.1/README.md +77 -0
- diffable_rdf-0.0.1/pyproject.toml +70 -0
- diffable_rdf-0.0.1/src/diffable_rdf/__init__.py +24 -0
- diffable_rdf-0.0.1/src/diffable_rdf/canonicalize.py +226 -0
- diffable_rdf-0.0.1/src/diffable_rdf/jsonld.py +50 -0
- diffable_rdf-0.0.1/src/diffable_rdf/py.typed +1 -0
- diffable_rdf-0.0.1/src/diffable_rdf/turtle.py +268 -0
- diffable_rdf-0.0.1/tests/test_diffable_rdf.py +132 -0
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*.egg-info/
|
|
5
|
+
.eggs/
|
|
6
|
+
build/
|
|
7
|
+
dist/
|
|
8
|
+
.venv/
|
|
9
|
+
venv/
|
|
10
|
+
|
|
11
|
+
# Tooling
|
|
12
|
+
.pytest_cache/
|
|
13
|
+
.ruff_cache/
|
|
14
|
+
.mypy_cache/
|
|
15
|
+
.coverage
|
|
16
|
+
htmlcov/
|
|
17
|
+
|
|
18
|
+
# Scratch (source extraction from upstream fork)
|
|
19
|
+
_src/
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
Apache License
|
|
2
|
+
Version 2.0, January 2004
|
|
3
|
+
http://www.apache.org/licenses/
|
|
4
|
+
|
|
5
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
6
|
+
|
|
7
|
+
1. Definitions.
|
|
8
|
+
|
|
9
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
|
10
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
|
11
|
+
|
|
12
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
|
13
|
+
the copyright owner that is granting the License.
|
|
14
|
+
|
|
15
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
|
16
|
+
other entities that control, are controlled by, or are under common
|
|
17
|
+
control with that entity. For the purposes of this definition,
|
|
18
|
+
"control" means (i) the power, direct or indirect, to cause the
|
|
19
|
+
direction or management of such entity, whether by contract or
|
|
20
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
21
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
22
|
+
|
|
23
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
|
24
|
+
exercising permissions granted by this License.
|
|
25
|
+
|
|
26
|
+
"Source" form shall mean the preferred form for making modifications,
|
|
27
|
+
including but not limited to software source code, documentation
|
|
28
|
+
source, and configuration files.
|
|
29
|
+
|
|
30
|
+
"Object" form shall mean any form resulting from mechanical
|
|
31
|
+
transformation or translation of a Source form, including but
|
|
32
|
+
not limited to compiled object code, generated documentation,
|
|
33
|
+
and conversions to other media types.
|
|
34
|
+
|
|
35
|
+
"Work" shall mean the work of authorship, whether in Source or
|
|
36
|
+
Object form, made available under the License, as indicated by a
|
|
37
|
+
copyright notice that is included in or attached to the work
|
|
38
|
+
(an example is provided in the Appendix below).
|
|
39
|
+
|
|
40
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
|
41
|
+
form, that is based on (or derived from) the Work and for which the
|
|
42
|
+
editorial revisions, annotations, elaborations, or other modifications
|
|
43
|
+
represent, as a whole, an original work of authorship. For the purposes
|
|
44
|
+
of this License, Derivative Works shall not include works that remain
|
|
45
|
+
separable from, or merely link (or bind by name) to the interfaces of,
|
|
46
|
+
the Work and Derivative Works thereof.
|
|
47
|
+
|
|
48
|
+
"Contribution" shall mean any work of authorship, including
|
|
49
|
+
the original version of the Work and any modifications or additions
|
|
50
|
+
to that Work or Derivative Works thereof, that is intentionally
|
|
51
|
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
|
52
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
|
53
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
|
54
|
+
means any form of electronic, verbal, or written communication sent
|
|
55
|
+
to the Licensor or its representatives, including but not limited to
|
|
56
|
+
communication on electronic mailing lists, source code control systems,
|
|
57
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
|
58
|
+
Licensor for the purpose of discussing and improving the Work, but
|
|
59
|
+
excluding communication that is conspicuously marked or otherwise
|
|
60
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
|
61
|
+
|
|
62
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
63
|
+
on behalf of whom a Contribution has been received by Licensor and
|
|
64
|
+
subsequently incorporated within the Work.
|
|
65
|
+
|
|
66
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
67
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
68
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
69
|
+
copyright license to reproduce, prepare Derivative Works of,
|
|
70
|
+
publicly display, publicly perform, sublicense, and distribute the
|
|
71
|
+
Work and such Derivative Works in Source or Object form.
|
|
72
|
+
|
|
73
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
|
74
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
75
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
76
|
+
(except as stated in this section) patent license to make, have made,
|
|
77
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
78
|
+
where such license applies only to those patent claims licensable
|
|
79
|
+
by such Contributor that are necessarily infringed by their
|
|
80
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
|
81
|
+
with the Work to which such Contribution(s) was submitted. If You
|
|
82
|
+
institute patent litigation against any entity (including a
|
|
83
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
|
84
|
+
or a Contribution incorporated within the Work constitutes direct
|
|
85
|
+
or contributory patent infringement, then any patent licenses
|
|
86
|
+
granted to You under this License for that Work shall terminate
|
|
87
|
+
as of the date such litigation is filed.
|
|
88
|
+
|
|
89
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
|
90
|
+
Work or Derivative Works thereof in any medium, with or without
|
|
91
|
+
modifications, and in Source or Object form, provided that You
|
|
92
|
+
meet the following conditions:
|
|
93
|
+
|
|
94
|
+
(a) You must give any other recipients of the Work or
|
|
95
|
+
Derivative Works a copy of this License; and
|
|
96
|
+
|
|
97
|
+
(b) You must cause any modified files to carry prominent notices
|
|
98
|
+
stating that You changed the files; and
|
|
99
|
+
|
|
100
|
+
(c) You must retain, in the Source form of any Derivative Works
|
|
101
|
+
that You distribute, all copyright, patent, trademark, and
|
|
102
|
+
attribution notices from the Source form of the Work,
|
|
103
|
+
excluding those notices that do not pertain to any part of
|
|
104
|
+
the Derivative Works; and
|
|
105
|
+
|
|
106
|
+
(d) If the Work includes a "NOTICE" text file as part of its
|
|
107
|
+
distribution, then any Derivative Works that You distribute must
|
|
108
|
+
include a readable copy of the attribution notices contained
|
|
109
|
+
within such NOTICE file, excluding those notices that do not
|
|
110
|
+
pertain to any part of the Derivative Works, in at least one
|
|
111
|
+
of the following places: within a NOTICE text file distributed
|
|
112
|
+
as part of the Derivative Works; within the Source form or
|
|
113
|
+
documentation, if provided along with the Derivative Works; or,
|
|
114
|
+
within a display generated by the Derivative Works, if and
|
|
115
|
+
wherever such third-party notices normally appear. The contents
|
|
116
|
+
of the NOTICE file are for informational purposes only and
|
|
117
|
+
do not modify the License. You may add Your own attribution
|
|
118
|
+
notices within Derivative Works that You distribute, alongside
|
|
119
|
+
or as an addendum to the NOTICE text from the Work, provided
|
|
120
|
+
that such additional attribution notices cannot be construed
|
|
121
|
+
as modifying the License.
|
|
122
|
+
|
|
123
|
+
You may add Your own copyright statement to Your modifications and
|
|
124
|
+
may provide additional or different license terms and conditions
|
|
125
|
+
for use, reproduction, or distribution of Your modifications, or
|
|
126
|
+
for any such Derivative Works as a whole, provided Your use,
|
|
127
|
+
reproduction, and distribution of the Work otherwise complies with
|
|
128
|
+
the conditions stated in this License.
|
|
129
|
+
|
|
130
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
131
|
+
any Contribution intentionally submitted for inclusion in the Work
|
|
132
|
+
by You to the Licensor shall be under the terms and conditions of
|
|
133
|
+
this License, without any additional terms or conditions.
|
|
134
|
+
Notwithstanding the above, nothing herein shall supersede or modify
|
|
135
|
+
the terms of any separate license agreement you may have executed
|
|
136
|
+
with Licensor regarding such Contributions.
|
|
137
|
+
|
|
138
|
+
6. Trademarks. This License does not grant permission to use the trade
|
|
139
|
+
names, trademarks, service marks, or product names of the Licensor,
|
|
140
|
+
except as required for reasonable and customary use in describing the
|
|
141
|
+
origin of the Work and reproducing the content of the NOTICE file.
|
|
142
|
+
|
|
143
|
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
|
144
|
+
agreed to in writing, Licensor provides the Work (and each
|
|
145
|
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
|
146
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
147
|
+
implied, including, without limitation, any warranties or conditions
|
|
148
|
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
|
149
|
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
|
150
|
+
appropriateness of using or redistributing the Work and assume any
|
|
151
|
+
risks associated with Your exercise of permissions under this License.
|
|
152
|
+
|
|
153
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
|
154
|
+
whether in tort (including negligence), contract, or otherwise,
|
|
155
|
+
unless required by applicable law (such as deliberate and grossly
|
|
156
|
+
negligent acts) or agreed to in writing, shall any Contributor be
|
|
157
|
+
liable to You for damages, including any direct, indirect, special,
|
|
158
|
+
incidental, or consequential damages of any character arising as a
|
|
159
|
+
result of this License or out of the use or inability to use the
|
|
160
|
+
Work (including but not limited to damages for loss of goodwill,
|
|
161
|
+
work stoppage, computer failure or malfunction, or any and all
|
|
162
|
+
other commercial damages or losses), even if such Contributor
|
|
163
|
+
has been advised of the possibility of such damages.
|
|
164
|
+
|
|
165
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
|
166
|
+
the Work or Derivative Works thereof, You may choose to offer,
|
|
167
|
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
|
168
|
+
or other liability obligations and/or rights consistent with this
|
|
169
|
+
License. However, in accepting such obligations, You may act only
|
|
170
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
|
171
|
+
of any other Contributor, and only if You agree to indemnify,
|
|
172
|
+
defend, and hold each Contributor harmless for any liability
|
|
173
|
+
incurred by, or claims asserted against, such Contributor by reason
|
|
174
|
+
of your accepting any such warranty or additional liability.
|
|
175
|
+
|
|
176
|
+
END OF TERMS AND CONDITIONS
|
|
177
|
+
|
|
178
|
+
APPENDIX: How to apply the Apache License to your work.
|
|
179
|
+
|
|
180
|
+
To apply the Apache License to your work, attach the following
|
|
181
|
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
|
182
|
+
replaced with your own identifying information. (Don't include
|
|
183
|
+
the brackets!) The text should be enclosed in the appropriate
|
|
184
|
+
comment syntax for the file format. We also recommend that a
|
|
185
|
+
file or class name and description of purpose be included on the
|
|
186
|
+
same "printed page" as the copyright notice for easier
|
|
187
|
+
identification within third-party archives.
|
|
188
|
+
|
|
189
|
+
Copyright [yyyy] [name of copyright owner]
|
|
190
|
+
|
|
191
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
192
|
+
you may not use this file except in compliance with the License.
|
|
193
|
+
You may obtain a copy of the License at
|
|
194
|
+
|
|
195
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
196
|
+
|
|
197
|
+
Unless required by applicable law or agreed to in writing, software
|
|
198
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
199
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
200
|
+
See the License for the specific language governing permissions and
|
|
201
|
+
limitations under the License.
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: diffable-rdf
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: Deterministic, diff-stable serialization for rdflib graphs (RDFC-1.0 + Weisfeiler-Lehman blank-node hashing + idiomatic Turtle).
|
|
5
|
+
Project-URL: Homepage, https://github.com/ASCS-eV/diffable-rdf
|
|
6
|
+
Project-URL: Repository, https://github.com/ASCS-eV/diffable-rdf
|
|
7
|
+
Project-URL: Issues, https://github.com/ASCS-eV/diffable-rdf/issues
|
|
8
|
+
Author: ASCS e.V.
|
|
9
|
+
License-Expression: Apache-2.0
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: canonicalization,deterministic,diff,json-ld,pyoxigraph,rdf,rdfc-1.0,rdflib,serialization,turtle
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
21
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
22
|
+
Classifier: Topic :: Text Processing :: Markup
|
|
23
|
+
Classifier: Typing :: Typed
|
|
24
|
+
Requires-Python: >=3.10
|
|
25
|
+
Requires-Dist: pyoxigraph>=0.4.0
|
|
26
|
+
Requires-Dist: rdflib>=6.0
|
|
27
|
+
Description-Content-Type: text/markdown
|
|
28
|
+
|
|
29
|
+
# diffable-rdf
|
|
30
|
+
|
|
31
|
+
Deterministic, **diff-stable** serialization for [rdflib](https://rdflib.readthedocs.io/)
|
|
32
|
+
graphs. Produces byte-identical Turtle across runs so version-controlled RDF
|
|
33
|
+
artifacts (OWL ontologies, SHACL shapes, JSON-LD contexts) show minimal,
|
|
34
|
+
meaningful diffs instead of blank-node churn.
|
|
35
|
+
|
|
36
|
+
## Why
|
|
37
|
+
|
|
38
|
+
RDF serializers assign blank-node identifiers (`_:c14nN`, `_:Nb1e2…`) based on
|
|
39
|
+
process-dependent ordering. Regenerating an ontology therefore produces large,
|
|
40
|
+
spurious diffs even when nothing semantically changed. `diffable-rdf` fixes this
|
|
41
|
+
with a standards-based pipeline:
|
|
42
|
+
|
|
43
|
+
1. **RDFC-1.0** ([W3C RDF Dataset Canonicalization](https://www.w3.org/TR/rdf-canon/))
|
|
44
|
+
via [pyoxigraph](https://pypi.org/project/pyoxigraph/) — isomorphic inputs
|
|
45
|
+
produce identical triple sets.
|
|
46
|
+
2. **Weisfeiler-Lehman structural hashing** — replaces sequential `_:c14nN`
|
|
47
|
+
identifiers with content-based hashes that depend only on graph structure,
|
|
48
|
+
so adding/removing a triple only touches the directly involved blank nodes.
|
|
49
|
+
3. **Idiomatic rdflib re-serialization** — inline blank nodes (`[ … ]`),
|
|
50
|
+
collection syntax (`( … )`), and filtered prefixes (only prefixes actually
|
|
51
|
+
used are declared).
|
|
52
|
+
|
|
53
|
+
All triples are preserved; only syntactic form changes.
|
|
54
|
+
|
|
55
|
+
## Install
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
pip install diffable-rdf
|
|
59
|
+
# or
|
|
60
|
+
uv add diffable-rdf
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
Requires Python 3.10+, `rdflib>=6`, and `pyoxigraph>=0.4`.
|
|
64
|
+
|
|
65
|
+
## Usage
|
|
66
|
+
|
|
67
|
+
```python
|
|
68
|
+
from rdflib import Graph
|
|
69
|
+
from diffable_rdf import deterministic_turtle
|
|
70
|
+
|
|
71
|
+
g = Graph().parse("ontology.ttl")
|
|
72
|
+
ttl = deterministic_turtle(g) # diff-stable, idiomatic Turtle
|
|
73
|
+
open("ontology.ttl", "w", newline="\n").write(ttl)
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
Other entry points:
|
|
77
|
+
|
|
78
|
+
```python
|
|
79
|
+
from diffable_rdf import canonicalize_rdf_graph, deterministic_json, well_known_prefix_map
|
|
80
|
+
|
|
81
|
+
canonicalize_rdf_graph(graph, "turtle") # lower-level RDFC-1.0 canonical form
|
|
82
|
+
deterministic_json(obj) # recursively key/list-sorted JSON(-LD)
|
|
83
|
+
well_known_prefix_map() # namespace IRI -> standard prefix name
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
## API
|
|
87
|
+
|
|
88
|
+
| Function | Purpose |
|
|
89
|
+
|---|---|
|
|
90
|
+
| `deterministic_turtle(graph) -> str` | Diff-stable, idiomatic Turtle (RDFC-1.0 + WL hashing + rdflib re-serialize). |
|
|
91
|
+
| `canonicalize_rdf_graph(graph, output_format="turtle") -> str` | RDFC-1.0 canonical serialization (with rdflib fallback for non-standard RDF). |
|
|
92
|
+
| `deterministic_json(obj, indent=3, preserve_list_order_keys=None) -> str` | Recursively sorted JSON; preserves JSON-LD ordered keys (`@context`, `@list`, …). |
|
|
93
|
+
| `well_known_prefix_map() -> dict[str, str]` | rdflib's curated namespace→prefix bindings. |
|
|
94
|
+
|
|
95
|
+
## Provenance
|
|
96
|
+
|
|
97
|
+
Extracted from the diff-stabilization work in
|
|
98
|
+
[`ASCS-eV/linkml#1`](https://github.com/ASCS-eV/linkml/pull/1) (itself a
|
|
99
|
+
review-ready rework of upstream [`linkml/linkml#3295`](https://github.com/linkml/linkml/pull/3295))
|
|
100
|
+
into a small, tool-agnostic library so LinkML, ShapeChange output, and other RDF
|
|
101
|
+
toolchains can share one canonicalizer.
|
|
102
|
+
|
|
103
|
+
## License
|
|
104
|
+
|
|
105
|
+
[Apache-2.0](LICENSE)
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
# diffable-rdf
|
|
2
|
+
|
|
3
|
+
Deterministic, **diff-stable** serialization for [rdflib](https://rdflib.readthedocs.io/)
|
|
4
|
+
graphs. Produces byte-identical Turtle across runs so version-controlled RDF
|
|
5
|
+
artifacts (OWL ontologies, SHACL shapes, JSON-LD contexts) show minimal,
|
|
6
|
+
meaningful diffs instead of blank-node churn.
|
|
7
|
+
|
|
8
|
+
## Why
|
|
9
|
+
|
|
10
|
+
RDF serializers assign blank-node identifiers (`_:c14nN`, `_:Nb1e2…`) based on
|
|
11
|
+
process-dependent ordering. Regenerating an ontology therefore produces large,
|
|
12
|
+
spurious diffs even when nothing semantically changed. `diffable-rdf` fixes this
|
|
13
|
+
with a standards-based pipeline:
|
|
14
|
+
|
|
15
|
+
1. **RDFC-1.0** ([W3C RDF Dataset Canonicalization](https://www.w3.org/TR/rdf-canon/))
|
|
16
|
+
via [pyoxigraph](https://pypi.org/project/pyoxigraph/) — isomorphic inputs
|
|
17
|
+
produce identical triple sets.
|
|
18
|
+
2. **Weisfeiler-Lehman structural hashing** — replaces sequential `_:c14nN`
|
|
19
|
+
identifiers with content-based hashes that depend only on graph structure,
|
|
20
|
+
so adding/removing a triple only touches the directly involved blank nodes.
|
|
21
|
+
3. **Idiomatic rdflib re-serialization** — inline blank nodes (`[ … ]`),
|
|
22
|
+
collection syntax (`( … )`), and filtered prefixes (only prefixes actually
|
|
23
|
+
used are declared).
|
|
24
|
+
|
|
25
|
+
All triples are preserved; only syntactic form changes.
|
|
26
|
+
|
|
27
|
+
## Install
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
pip install diffable-rdf
|
|
31
|
+
# or
|
|
32
|
+
uv add diffable-rdf
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
Requires Python 3.10+, `rdflib>=6`, and `pyoxigraph>=0.4`.
|
|
36
|
+
|
|
37
|
+
## Usage
|
|
38
|
+
|
|
39
|
+
```python
|
|
40
|
+
from rdflib import Graph
|
|
41
|
+
from diffable_rdf import deterministic_turtle
|
|
42
|
+
|
|
43
|
+
g = Graph().parse("ontology.ttl")
|
|
44
|
+
ttl = deterministic_turtle(g) # diff-stable, idiomatic Turtle
|
|
45
|
+
open("ontology.ttl", "w", newline="\n").write(ttl)
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
Other entry points:
|
|
49
|
+
|
|
50
|
+
```python
|
|
51
|
+
from diffable_rdf import canonicalize_rdf_graph, deterministic_json, well_known_prefix_map
|
|
52
|
+
|
|
53
|
+
canonicalize_rdf_graph(graph, "turtle") # lower-level RDFC-1.0 canonical form
|
|
54
|
+
deterministic_json(obj) # recursively key/list-sorted JSON(-LD)
|
|
55
|
+
well_known_prefix_map() # namespace IRI -> standard prefix name
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
## API
|
|
59
|
+
|
|
60
|
+
| Function | Purpose |
|
|
61
|
+
|---|---|
|
|
62
|
+
| `deterministic_turtle(graph) -> str` | Diff-stable, idiomatic Turtle (RDFC-1.0 + WL hashing + rdflib re-serialize). |
|
|
63
|
+
| `canonicalize_rdf_graph(graph, output_format="turtle") -> str` | RDFC-1.0 canonical serialization (with rdflib fallback for non-standard RDF). |
|
|
64
|
+
| `deterministic_json(obj, indent=3, preserve_list_order_keys=None) -> str` | Recursively sorted JSON; preserves JSON-LD ordered keys (`@context`, `@list`, …). |
|
|
65
|
+
| `well_known_prefix_map() -> dict[str, str]` | rdflib's curated namespace→prefix bindings. |
|
|
66
|
+
|
|
67
|
+
## Provenance
|
|
68
|
+
|
|
69
|
+
Extracted from the diff-stabilization work in
|
|
70
|
+
[`ASCS-eV/linkml#1`](https://github.com/ASCS-eV/linkml/pull/1) (itself a
|
|
71
|
+
review-ready rework of upstream [`linkml/linkml#3295`](https://github.com/linkml/linkml/pull/3295))
|
|
72
|
+
into a small, tool-agnostic library so LinkML, ShapeChange output, and other RDF
|
|
73
|
+
toolchains can share one canonicalizer.
|
|
74
|
+
|
|
75
|
+
## License
|
|
76
|
+
|
|
77
|
+
[Apache-2.0](LICENSE)
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "diffable-rdf"
|
|
7
|
+
dynamic = ["version"]
|
|
8
|
+
description = "Deterministic, diff-stable serialization for rdflib graphs (RDFC-1.0 + Weisfeiler-Lehman blank-node hashing + idiomatic Turtle)."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = "Apache-2.0"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [{ name = "ASCS e.V." }]
|
|
14
|
+
keywords = [
|
|
15
|
+
"rdf",
|
|
16
|
+
"rdflib",
|
|
17
|
+
"turtle",
|
|
18
|
+
"canonicalization",
|
|
19
|
+
"rdfc-1.0",
|
|
20
|
+
"deterministic",
|
|
21
|
+
"diff",
|
|
22
|
+
"serialization",
|
|
23
|
+
"json-ld",
|
|
24
|
+
"pyoxigraph",
|
|
25
|
+
]
|
|
26
|
+
classifiers = [
|
|
27
|
+
"Development Status :: 3 - Alpha",
|
|
28
|
+
"Intended Audience :: Developers",
|
|
29
|
+
"License :: OSI Approved :: Apache Software License",
|
|
30
|
+
"Operating System :: OS Independent",
|
|
31
|
+
"Programming Language :: Python :: 3",
|
|
32
|
+
"Programming Language :: Python :: 3.10",
|
|
33
|
+
"Programming Language :: Python :: 3.11",
|
|
34
|
+
"Programming Language :: Python :: 3.12",
|
|
35
|
+
"Programming Language :: Python :: 3.13",
|
|
36
|
+
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
37
|
+
"Topic :: Text Processing :: Markup",
|
|
38
|
+
"Typing :: Typed",
|
|
39
|
+
]
|
|
40
|
+
dependencies = [
|
|
41
|
+
"rdflib>=6.0",
|
|
42
|
+
"pyoxigraph>=0.4.0",
|
|
43
|
+
]
|
|
44
|
+
|
|
45
|
+
[project.urls]
|
|
46
|
+
Homepage = "https://github.com/ASCS-eV/diffable-rdf"
|
|
47
|
+
Repository = "https://github.com/ASCS-eV/diffable-rdf"
|
|
48
|
+
Issues = "https://github.com/ASCS-eV/diffable-rdf/issues"
|
|
49
|
+
|
|
50
|
+
[dependency-groups]
|
|
51
|
+
dev = [
|
|
52
|
+
"pytest>=8.0",
|
|
53
|
+
]
|
|
54
|
+
|
|
55
|
+
[tool.hatch.version]
|
|
56
|
+
path = "src/diffable_rdf/__init__.py"
|
|
57
|
+
|
|
58
|
+
[tool.hatch.build.targets.wheel]
|
|
59
|
+
packages = ["src/diffable_rdf"]
|
|
60
|
+
|
|
61
|
+
[tool.hatch.build.targets.sdist]
|
|
62
|
+
include = [
|
|
63
|
+
"src/diffable_rdf",
|
|
64
|
+
"tests",
|
|
65
|
+
"README.md",
|
|
66
|
+
"LICENSE",
|
|
67
|
+
]
|
|
68
|
+
|
|
69
|
+
[tool.pytest.ini_options]
|
|
70
|
+
testpaths = ["tests"]
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"""diffable-rdf: deterministic, diff-stable serialization for rdflib graphs.
|
|
2
|
+
|
|
3
|
+
Public API:
|
|
4
|
+
deterministic_turtle(graph) -> diff-stable idiomatic Turtle
|
|
5
|
+
canonicalize_rdf_graph(graph, format) -> RDFC-1.0 canonical serialization
|
|
6
|
+
deterministic_json(obj) -> deterministically ordered JSON
|
|
7
|
+
well_known_prefix_map() -> namespace IRI -> standard prefix
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from diffable_rdf.canonicalize import canonicalize_rdf_graph
|
|
13
|
+
from diffable_rdf.jsonld import deterministic_json
|
|
14
|
+
from diffable_rdf.turtle import deterministic_turtle, well_known_prefix_map
|
|
15
|
+
|
|
16
|
+
__all__ = [
|
|
17
|
+
"deterministic_turtle",
|
|
18
|
+
"canonicalize_rdf_graph",
|
|
19
|
+
"deterministic_json",
|
|
20
|
+
"well_known_prefix_map",
|
|
21
|
+
"__version__",
|
|
22
|
+
]
|
|
23
|
+
|
|
24
|
+
__version__ = "0.0.1"
|
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
"""Deterministic RDF serialization via pyoxigraph RDFC-1.0 canonicalization.
|
|
2
|
+
|
|
3
|
+
This module provides a function to canonicalize an rdflib Graph using
|
|
4
|
+
pyoxigraph's RDFC-1.0 implementation, producing deterministic output
|
|
5
|
+
with stable blank node labels and sorted triples.
|
|
6
|
+
|
|
7
|
+
**Known limitations:**
|
|
8
|
+
|
|
9
|
+
1. **xsd:string normalization**: pyoxigraph follows RDF 1.1, where plain
|
|
10
|
+
string literals and ``"text"^^xsd:string`` are identical. The output
|
|
11
|
+
will never contain explicit ``^^xsd:string`` annotations. Code that
|
|
12
|
+
re-parses the output with rdflib will see ``Literal("x")`` (datatype
|
|
13
|
+
``None``) rather than ``Literal("x", datatype=XSD.string)``.
|
|
14
|
+
|
|
15
|
+
2. **Non-standard RDF**: Graphs with literal predicates (e.g. SHACL
|
|
16
|
+
annotation mode) are rejected by pyoxigraph. This function falls
|
|
17
|
+
back to rdflib's serializer for such graphs.
|
|
18
|
+
|
|
19
|
+
3. **Numeric short forms**: pyoxigraph uses Turtle short forms for
|
|
20
|
+
``xsd:integer`` (``42``), ``xsd:boolean`` (``true``), and
|
|
21
|
+
``xsd:decimal`` (``1.23``). rdflib parses these back with the
|
|
22
|
+
correct datatype, so this is lossless.
|
|
23
|
+
|
|
24
|
+
4. **Base IRI / prefix collision**: When a graph has ``@base`` and a
|
|
25
|
+
prefix whose namespace equals the base IRI (e.g. rdflib's auto-bound
|
|
26
|
+
``base:`` prefix), pyoxigraph emits CURIEs like ``base:label`` that
|
|
27
|
+
rdflib rejects. We skip such prefixes during serialization.
|
|
28
|
+
|
|
29
|
+
5. **Trailing escaped dot in PN_LOCAL**: pyoxigraph emits CURIEs like
|
|
30
|
+
``prefix:local\\.`` for IRIs whose local part ends with ``.``. This
|
|
31
|
+
is valid Turtle (PN_LOCAL_ESC), but rdflib's notation3 parser rejects
|
|
32
|
+
it because it conflicts with the statement-terminator dot. We
|
|
33
|
+
post-process the output to expand such CURIEs to full ``<IRI>`` form.
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
import io
|
|
37
|
+
import logging
|
|
38
|
+
import re
|
|
39
|
+
|
|
40
|
+
import pyoxigraph as ox
|
|
41
|
+
import rdflib
|
|
42
|
+
|
|
43
|
+
logger = logging.getLogger(__name__)
|
|
44
|
+
|
|
45
|
+
# Mapping from rdflib format strings to pyoxigraph RdfFormat objects.
|
|
46
|
+
_FORMAT_MAP: dict[str, ox.RdfFormat] = {
|
|
47
|
+
"turtle": ox.RdfFormat.TURTLE,
|
|
48
|
+
"ttl": ox.RdfFormat.TURTLE,
|
|
49
|
+
"nt": ox.RdfFormat.N_TRIPLES,
|
|
50
|
+
"ntriples": ox.RdfFormat.N_TRIPLES,
|
|
51
|
+
"n-triples": ox.RdfFormat.N_TRIPLES,
|
|
52
|
+
"nt11": ox.RdfFormat.N_TRIPLES,
|
|
53
|
+
"nquads": ox.RdfFormat.N_QUADS,
|
|
54
|
+
"n-quads": ox.RdfFormat.N_QUADS,
|
|
55
|
+
"xml": ox.RdfFormat.RDF_XML,
|
|
56
|
+
"rdf/xml": ox.RdfFormat.RDF_XML,
|
|
57
|
+
"trig": ox.RdfFormat.TRIG,
|
|
58
|
+
"n3": ox.RdfFormat.N3,
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
# Formats that support prefix declarations.
|
|
62
|
+
_PREFIX_FORMATS = frozenset({ox.RdfFormat.TURTLE, ox.RdfFormat.TRIG, ox.RdfFormat.N3, ox.RdfFormat.RDF_XML})
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
# Characters that may appear escaped in a Turtle PN_LOCAL via PN_LOCAL_ESC.
|
|
66
|
+
_PN_LOCAL_ESC_UNESCAPE = re.compile(r"\\([_~.\-!$&'()*+,;=/?#@%])")
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _expand_trailing_dot_curies(turtle_text: str, prefixes: dict[str, str]) -> str:
|
|
70
|
+
"""Replace CURIEs whose local part ends in ``\\.`` with full ``<IRI>`` form.
|
|
71
|
+
|
|
72
|
+
rdflib's notation3 parser rejects PN_LOCAL ending in an escaped dot
|
|
73
|
+
even though Turtle permits it (PN_LOCAL_ESC). pyoxigraph emits this
|
|
74
|
+
form for IRIs ending in ``.`` (e.g. ``biolink:StrandEnum#.``). We
|
|
75
|
+
rewrite each such CURIE to its expanded ``<IRI>`` form so the output
|
|
76
|
+
round-trips through rdflib.
|
|
77
|
+
"""
|
|
78
|
+
if not prefixes:
|
|
79
|
+
return turtle_text
|
|
80
|
+
|
|
81
|
+
# Match: a prefix name, ':', a local part (no whitespace or token
|
|
82
|
+
# delimiters), ending in ``\.``, followed by whitespace. Use a
|
|
83
|
+
# negative lookbehind to avoid matching inside ``<...>`` or word
|
|
84
|
+
# characters that would make this a substring of something else.
|
|
85
|
+
pattern = re.compile(
|
|
86
|
+
r"(?<![<\w])"
|
|
87
|
+
r"([A-Za-z_][\w.-]*?):"
|
|
88
|
+
r"([^\s,;()<>\"'\[\]]*?\\\.)"
|
|
89
|
+
r"(?=\s)"
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
def replace(match: re.Match[str]) -> str:
|
|
93
|
+
prefix = match.group(1)
|
|
94
|
+
local_escaped = match.group(2)
|
|
95
|
+
namespace = prefixes.get(prefix)
|
|
96
|
+
if namespace is None:
|
|
97
|
+
return match.group(0)
|
|
98
|
+
local = _PN_LOCAL_ESC_UNESCAPE.sub(r"\1", local_escaped)
|
|
99
|
+
return f"<{namespace}{local}>"
|
|
100
|
+
|
|
101
|
+
return pattern.sub(replace, turtle_text)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _is_safe_prefix_iri(iri: str) -> bool:
|
|
105
|
+
"""Check whether a namespace IRI is safe for prefix serialization.
|
|
106
|
+
|
|
107
|
+
pyoxigraph rejects IRIs with invalid code-points (e.g. double ``#``),
|
|
108
|
+
and rdflib's Turtle parser cannot round-trip CURIEs whose namespace
|
|
109
|
+
contains query parameters or fragments in unexpected positions. This
|
|
110
|
+
function returns ``False`` for such IRIs so they can be skipped during
|
|
111
|
+
prefix collection.
|
|
112
|
+
"""
|
|
113
|
+
# A namespace IRI should end with '/' or '#'. If '#' appears
|
|
114
|
+
# *before* the final character, the IRI contains an embedded
|
|
115
|
+
# fragment which produces unusable CURIEs.
|
|
116
|
+
if "#" in iri[:-1]:
|
|
117
|
+
return False
|
|
118
|
+
# Query parameters in namespace IRIs produce CURIEs that rdflib
|
|
119
|
+
# cannot parse back.
|
|
120
|
+
if "?" in iri:
|
|
121
|
+
return False
|
|
122
|
+
return True
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def canonicalize_rdf_graph(
|
|
126
|
+
graph: rdflib.Graph,
|
|
127
|
+
output_format: str = "turtle",
|
|
128
|
+
) -> str:
|
|
129
|
+
"""Serialize an rdflib Graph deterministically using RDFC-1.0 canonicalization.
|
|
130
|
+
|
|
131
|
+
The graph is transferred to pyoxigraph via N-Triples, canonicalized
|
|
132
|
+
with RDFC-1.0, sorted, and serialized back to the requested format.
|
|
133
|
+
Prefix bindings from the rdflib Graph are preserved in the output
|
|
134
|
+
for formats that support them (Turtle, TriG, N3, RDF/XML).
|
|
135
|
+
|
|
136
|
+
Falls back to plain rdflib serialization for unsupported formats or
|
|
137
|
+
graphs containing non-standard RDF (e.g. literal predicates).
|
|
138
|
+
|
|
139
|
+
:param graph: The rdflib Graph to serialize.
|
|
140
|
+
:param output_format: Target serialization format (e.g. ``"turtle"``, ``"nt"``).
|
|
141
|
+
:return: Deterministic string serialization of the graph.
|
|
142
|
+
"""
|
|
143
|
+
ox_format = _FORMAT_MAP.get(output_format.lower())
|
|
144
|
+
if ox_format is None:
|
|
145
|
+
logger.warning(
|
|
146
|
+
"pyoxigraph does not support format %r; falling back to rdflib serializer",
|
|
147
|
+
output_format,
|
|
148
|
+
)
|
|
149
|
+
# rdflib's Turtle serializer emits a trailing double newline;
|
|
150
|
+
# normalize to single newline for consistent file endings.
|
|
151
|
+
data = graph.serialize(format=output_format)
|
|
152
|
+
return data.rstrip("\n") + "\n" if data.endswith("\n") else data
|
|
153
|
+
|
|
154
|
+
# 1. Transfer rdflib graph to pyoxigraph via N-Triples.
|
|
155
|
+
nt_data = graph.serialize(format="nt")
|
|
156
|
+
nt_bytes = nt_data.encode("utf-8") if isinstance(nt_data, str) else nt_data
|
|
157
|
+
|
|
158
|
+
# 2. Parse into pyoxigraph and build a Dataset for canonicalization.
|
|
159
|
+
# Fall back to rdflib if the graph contains non-standard RDF
|
|
160
|
+
# (e.g. literal predicates from annotations) that pyoxigraph rejects.
|
|
161
|
+
try:
|
|
162
|
+
triples = list(ox.parse(io.BytesIO(nt_bytes), format=ox.RdfFormat.N_TRIPLES))
|
|
163
|
+
except SyntaxError:
|
|
164
|
+
logger.warning(
|
|
165
|
+
"Graph contains non-standard RDF that pyoxigraph cannot parse; falling back to rdflib serializer"
|
|
166
|
+
)
|
|
167
|
+
return graph.serialize(format=output_format)
|
|
168
|
+
|
|
169
|
+
dataset = ox.Dataset()
|
|
170
|
+
for triple in triples:
|
|
171
|
+
dataset.add(ox.Quad(triple.subject, triple.predicate, triple.object, ox.DefaultGraph()))
|
|
172
|
+
|
|
173
|
+
# 3. Canonicalize blank node labels with RDFC-1.0.
|
|
174
|
+
dataset.canonicalize(ox.CanonicalizationAlgorithm.RDFC_1_0)
|
|
175
|
+
|
|
176
|
+
# 4. Sort triples for deterministic ordering.
|
|
177
|
+
quads = list(dataset)
|
|
178
|
+
sorted_triples = sorted(
|
|
179
|
+
(ox.Triple(q.subject, q.predicate, q.object) for q in quads),
|
|
180
|
+
key=lambda t: (str(t.subject), str(t.predicate), str(t.object)),
|
|
181
|
+
)
|
|
182
|
+
|
|
183
|
+
# 5. Collect prefixes for formats that support them.
|
|
184
|
+
base_iri = str(graph.base) if graph.base else None
|
|
185
|
+
prefixes: dict[str, str] | None = None
|
|
186
|
+
if ox_format in _PREFIX_FORMATS:
|
|
187
|
+
prefixes = {}
|
|
188
|
+
for prefix, namespace in graph.namespace_manager.namespaces():
|
|
189
|
+
if not prefix: # skip empty prefix (base)
|
|
190
|
+
continue
|
|
191
|
+
ns_str = str(namespace)
|
|
192
|
+
# Skip prefixes whose namespace matches the base IRI to avoid
|
|
193
|
+
# pyoxigraph emitting CURIEs like `base:label` that conflict
|
|
194
|
+
# with the @base directive.
|
|
195
|
+
if base_iri and ns_str == base_iri:
|
|
196
|
+
continue
|
|
197
|
+
# Skip namespace IRIs that pyoxigraph rejects or that produce
|
|
198
|
+
# CURIEs rdflib cannot round-trip. Valid namespace IRIs for
|
|
199
|
+
# prefix use should end with '/' or '#' and contain no query
|
|
200
|
+
# parameters or fragment-like characters in the middle.
|
|
201
|
+
if not _is_safe_prefix_iri(ns_str):
|
|
202
|
+
continue
|
|
203
|
+
prefixes[str(prefix)] = ns_str
|
|
204
|
+
used_prefixes = prefixes
|
|
205
|
+
try:
|
|
206
|
+
result_bytes = ox.serialize(
|
|
207
|
+
sorted_triples,
|
|
208
|
+
format=ox_format,
|
|
209
|
+
prefixes=prefixes,
|
|
210
|
+
base_iri=base_iri,
|
|
211
|
+
)
|
|
212
|
+
except ValueError:
|
|
213
|
+
# pyoxigraph rejects prefixes with invalid IRIs (e.g. containing
|
|
214
|
+
# fragment-like characters such as double '#'). Retry without
|
|
215
|
+
# the offending prefixes by falling back to no prefixes, which
|
|
216
|
+
# still produces valid (if verbose) Turtle.
|
|
217
|
+
logger.warning("pyoxigraph rejected one or more prefix IRIs; serializing without prefix declarations")
|
|
218
|
+
result_bytes = ox.serialize(
|
|
219
|
+
sorted_triples,
|
|
220
|
+
format=ox_format,
|
|
221
|
+
)
|
|
222
|
+
used_prefixes = None
|
|
223
|
+
result = result_bytes.decode("utf-8")
|
|
224
|
+
if ox_format in _PREFIX_FORMATS and used_prefixes:
|
|
225
|
+
result = _expand_trailing_dot_curies(result, used_prefixes)
|
|
226
|
+
return result
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
"""Deterministic JSON / JSON-LD serialization (recursive key + list sort)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
|
|
7
|
+
# JSON-LD keys whose array values carry ordering semantics and must NOT be
|
|
8
|
+
# sorted. ``@context`` arrays define an override cascade (JSON-LD 1.1 §4.1);
|
|
9
|
+
# ``@list`` containers are explicitly ordered; ``@graph``/``@set`` and
|
|
10
|
+
# ``imports`` are included defensively.
|
|
11
|
+
_JSONLD_ORDERED_KEYS: frozenset[str] = frozenset({"@context", "@list", "@graph", "@set", "imports"})
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def deterministic_json(
|
|
15
|
+
obj: object,
|
|
16
|
+
indent: int = 3,
|
|
17
|
+
preserve_list_order_keys: frozenset[str] | None = None,
|
|
18
|
+
) -> str:
|
|
19
|
+
"""Serialize a JSON-compatible object with deterministic ordering.
|
|
20
|
+
|
|
21
|
+
Recursively sorts all dict keys *and* list elements to produce stable
|
|
22
|
+
output across Python versions and process invocations.
|
|
23
|
+
|
|
24
|
+
List elements are sorted by their canonical JSON representation
|
|
25
|
+
(``json.dumps(item, sort_keys=True)``), which handles lists of dicts,
|
|
26
|
+
strings, and mixed types.
|
|
27
|
+
|
|
28
|
+
:param obj: A JSON-serializable object.
|
|
29
|
+
:param indent: Number of spaces for indentation.
|
|
30
|
+
:param preserve_list_order_keys: Dict keys whose list values must NOT be
|
|
31
|
+
sorted (e.g. ``@context``, ``@list`` in JSON-LD where array order is
|
|
32
|
+
semantic). Defaults to :data:`_JSONLD_ORDERED_KEYS`.
|
|
33
|
+
:returns: Deterministic JSON string.
|
|
34
|
+
"""
|
|
35
|
+
skip = preserve_list_order_keys if preserve_list_order_keys is not None else _JSONLD_ORDERED_KEYS
|
|
36
|
+
|
|
37
|
+
def _deep_sort(value: object, parent_key: str = "") -> object:
|
|
38
|
+
if isinstance(value, dict):
|
|
39
|
+
return {k: _deep_sort(v, parent_key=k) for k, v in sorted(value.items())}
|
|
40
|
+
if isinstance(value, list):
|
|
41
|
+
sorted_items = [_deep_sort(item) for item in value]
|
|
42
|
+
if parent_key in skip:
|
|
43
|
+
return sorted_items
|
|
44
|
+
try:
|
|
45
|
+
return sorted(sorted_items, key=lambda x: json.dumps(x, sort_keys=True, ensure_ascii=False))
|
|
46
|
+
except TypeError:
|
|
47
|
+
return sorted_items
|
|
48
|
+
return value
|
|
49
|
+
|
|
50
|
+
return json.dumps(_deep_sort(obj), indent=indent, ensure_ascii=False)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
# Marker file (PEP 561): ship inline type hints to consumers.
|
|
@@ -0,0 +1,268 @@
|
|
|
1
|
+
"""Deterministic, diff-stable Turtle serialization for rdflib graphs.
|
|
2
|
+
|
|
3
|
+
Three-phase hybrid pipeline: RDFC-1.0 canonicalization (pyoxigraph) ->
|
|
4
|
+
Weisfeiler-Lehman blank-node hashing -> idiomatic rdflib re-serialization.
|
|
5
|
+
|
|
6
|
+
Extracted from ASCS-eV/linkml (feat/deterministic-output, PR #1) into a
|
|
7
|
+
standalone, tool-agnostic library.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import logging
|
|
13
|
+
from typing import TYPE_CHECKING
|
|
14
|
+
|
|
15
|
+
if TYPE_CHECKING:
|
|
16
|
+
from rdflib import Graph as RdfGraph
|
|
17
|
+
|
|
18
|
+
logger = logging.getLogger(__name__)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _wl_signatures(
|
|
22
|
+
quads: list,
|
|
23
|
+
iterations: int = 4,
|
|
24
|
+
) -> dict[str, str]:
|
|
25
|
+
"""Compute Weisfeiler-Lehman structural signatures for blank nodes.
|
|
26
|
+
|
|
27
|
+
Uses 1-dimensional WL colour refinement [1]_ to assign each blank
|
|
28
|
+
node a deterministic signature derived from its multi-hop
|
|
29
|
+
neighbourhood structure. The signature depends only on predicate
|
|
30
|
+
IRIs, literal values, and named-node IRIs — **not** on blank-node
|
|
31
|
+
identifiers — so it remains stable when unrelated triples are added
|
|
32
|
+
or removed.
|
|
33
|
+
|
|
34
|
+
Parameters
|
|
35
|
+
----------
|
|
36
|
+
quads : list
|
|
37
|
+
Canonical quads from pyoxigraph (after RDFC-1.0).
|
|
38
|
+
iterations : int
|
|
39
|
+
Number of WL refinement rounds (default 4).
|
|
40
|
+
|
|
41
|
+
Returns
|
|
42
|
+
-------
|
|
43
|
+
dict[str, str]
|
|
44
|
+
Mapping from canonical blank-node ID (e.g. ``c14n42``) to a
|
|
45
|
+
truncated SHA-256 hash suitable for use as a stable blank-node
|
|
46
|
+
label.
|
|
47
|
+
|
|
48
|
+
References
|
|
49
|
+
----------
|
|
50
|
+
.. [1] Weisfeiler, B. & Leman, A. (1968). "The reduction of a graph
|
|
51
|
+
to canonical form and the algebra which appears therein."
|
|
52
|
+
"""
|
|
53
|
+
import hashlib
|
|
54
|
+
|
|
55
|
+
import pyoxigraph # guaranteed available — caller (deterministic_turtle) checks
|
|
56
|
+
|
|
57
|
+
# Collect all blank node IDs and build adjacency index.
|
|
58
|
+
bnode_ids: set[str] = set()
|
|
59
|
+
# outgoing[b] = list of (predicate_str, object_str_or_bnode_id, is_bnode)
|
|
60
|
+
outgoing: dict[str, list[tuple[str, str, bool]]] = {}
|
|
61
|
+
# incoming[b] = list of (subject_str_or_bnode_id, predicate_str, is_bnode)
|
|
62
|
+
incoming: dict[str, list[tuple[str, str, bool]]] = {}
|
|
63
|
+
|
|
64
|
+
for q in quads:
|
|
65
|
+
s, p, o = q.subject, q.predicate, q.object
|
|
66
|
+
s_is_bn = isinstance(s, pyoxigraph.BlankNode)
|
|
67
|
+
o_is_bn = isinstance(o, pyoxigraph.BlankNode)
|
|
68
|
+
p_str = str(p)
|
|
69
|
+
|
|
70
|
+
if s_is_bn:
|
|
71
|
+
bnode_ids.add(s.value)
|
|
72
|
+
outgoing.setdefault(s.value, []).append((p_str, o.value if o_is_bn else str(o), o_is_bn))
|
|
73
|
+
if o_is_bn:
|
|
74
|
+
bnode_ids.add(o.value)
|
|
75
|
+
incoming.setdefault(o.value, []).append((s.value if s_is_bn else str(s), p_str, s_is_bn))
|
|
76
|
+
|
|
77
|
+
# Initialise signatures: named-node edges only (no bnode IDs).
|
|
78
|
+
sig: dict[str, str] = {}
|
|
79
|
+
for bid in bnode_ids:
|
|
80
|
+
parts = []
|
|
81
|
+
for p_str, o_str, o_is_bn in outgoing.get(bid, []):
|
|
82
|
+
if not o_is_bn:
|
|
83
|
+
parts.append(f"+{p_str}={o_str}")
|
|
84
|
+
for s_str, p_str, s_is_bn in incoming.get(bid, []):
|
|
85
|
+
if not s_is_bn:
|
|
86
|
+
parts.append(f"-{s_str}={p_str}")
|
|
87
|
+
sig[bid] = "|".join(sorted(parts))
|
|
88
|
+
|
|
89
|
+
# Iterative refinement: incorporate neighbour signatures.
|
|
90
|
+
for _ in range(iterations):
|
|
91
|
+
new_sig: dict[str, str] = {}
|
|
92
|
+
for bid in bnode_ids:
|
|
93
|
+
parts = [sig[bid]]
|
|
94
|
+
for p_str, o_str, o_is_bn in outgoing.get(bid, []):
|
|
95
|
+
if o_is_bn:
|
|
96
|
+
parts.append(f"+{p_str}={sig.get(o_str, '')}")
|
|
97
|
+
for s_str, p_str, s_is_bn in incoming.get(bid, []):
|
|
98
|
+
if s_is_bn:
|
|
99
|
+
parts.append(f"-{sig.get(s_str, '')}={p_str}")
|
|
100
|
+
new_sig[bid] = "|".join(sorted(parts))
|
|
101
|
+
sig = new_sig
|
|
102
|
+
|
|
103
|
+
# Convert signatures to truncated SHA-256 hashes.
|
|
104
|
+
# Use 12 hex chars (48 bits) — birthday-bound collision probability
|
|
105
|
+
# is ~n²/2^49: ~0.002% at 100k nodes. Collisions are handled by
|
|
106
|
+
# appending a counter (see below), so correctness is preserved.
|
|
107
|
+
hash_map: dict[str, str] = {}
|
|
108
|
+
seen_hashes: dict[str, int] = {}
|
|
109
|
+
for bid in sorted(bnode_ids):
|
|
110
|
+
digest = hashlib.sha256(sig[bid].encode("utf-8")).hexdigest()[:12]
|
|
111
|
+
# Handle collisions by appending a counter.
|
|
112
|
+
count = seen_hashes.get(digest, 0)
|
|
113
|
+
seen_hashes[digest] = count + 1
|
|
114
|
+
label = f"b{digest}" if count == 0 else f"b{digest}_{count}"
|
|
115
|
+
hash_map[bid] = label
|
|
116
|
+
|
|
117
|
+
return hash_map
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def deterministic_turtle(graph: "RdfGraph") -> str:
|
|
123
|
+
"""Serialize an RDF graph to Turtle with deterministic output ordering.
|
|
124
|
+
|
|
125
|
+
Uses a three-phase hybrid pipeline for **correctness**, **diff
|
|
126
|
+
stability**, and **readability**:
|
|
127
|
+
|
|
128
|
+
1. **RDFC-1.0** [1]_ (via ``pyoxigraph``) canonicalizes the graph,
|
|
129
|
+
ensuring isomorphic inputs produce identical triple sets.
|
|
130
|
+
2. **Weisfeiler-Lehman structural hashing** replaces the sequential
|
|
131
|
+
``_:c14nN`` identifiers with content-based hashes derived from
|
|
132
|
+
each blank node's multi-hop neighbourhood. These hashes depend
|
|
133
|
+
only on predicate IRIs, literal values, and named-node IRIs —
|
|
134
|
+
not on blank-node numbering — so adding or removing a triple
|
|
135
|
+
only affects the identifiers of directly involved blank nodes.
|
|
136
|
+
3. **Hybrid rdflib re-serialization** parses the canonicalized,
|
|
137
|
+
WL-hashed triples back into an rdflib ``Graph`` and serializes
|
|
138
|
+
with rdflib's native Turtle writer. This recovers idiomatic
|
|
139
|
+
Turtle features that pyoxigraph cannot emit:
|
|
140
|
+
|
|
141
|
+
- **Inline blank nodes** (``[ … ]``) for singly-referenced
|
|
142
|
+
blank nodes (Turtle §2.7 [2]_), instead of verbose named
|
|
143
|
+
``_:bHASH`` syntax.
|
|
144
|
+
- **Collection syntax** (``( … )``) for ``rdf:List`` chains
|
|
145
|
+
(Turtle §2.8 [2]_).
|
|
146
|
+
- **Prefix filtering**: only prefixes actually used in the
|
|
147
|
+
graph's IRIs are declared, following the practice of Apache
|
|
148
|
+
Jena, Eclipse RDF4J, and Raptor.
|
|
149
|
+
|
|
150
|
+
All triples from the source graph are preserved — the hybrid step
|
|
151
|
+
only changes syntactic form, never semantic content.
|
|
152
|
+
|
|
153
|
+
Parameters
|
|
154
|
+
----------
|
|
155
|
+
graph : rdflib.Graph
|
|
156
|
+
An rdflib Graph to serialize.
|
|
157
|
+
|
|
158
|
+
Returns
|
|
159
|
+
-------
|
|
160
|
+
str
|
|
161
|
+
Deterministic Turtle string with ``@prefix`` declarations.
|
|
162
|
+
|
|
163
|
+
References
|
|
164
|
+
----------
|
|
165
|
+
.. [1] W3C (2024). "RDF Dataset Canonicalization (RDFC-1.0)."
|
|
166
|
+
W3C Recommendation. https://www.w3.org/TR/rdf-canon/
|
|
167
|
+
.. [2] W3C (2014). "RDF 1.1 Turtle — Terse RDF Triple Language."
|
|
168
|
+
W3C Recommendation. https://www.w3.org/TR/turtle/
|
|
169
|
+
"""
|
|
170
|
+
try:
|
|
171
|
+
import pyoxigraph
|
|
172
|
+
except ImportError as exc:
|
|
173
|
+
raise ImportError(
|
|
174
|
+
"pyoxigraph >= 0.4.0 is required for --deterministic output. "
|
|
175
|
+
"Install it with: pip install 'pyoxigraph>=0.4.0'"
|
|
176
|
+
) from exc
|
|
177
|
+
|
|
178
|
+
from rdflib import BNode, Graph, Literal, URIRef
|
|
179
|
+
|
|
180
|
+
# ── Phase 1: RDFC-1.0 canonicalization ──────────────────────────
|
|
181
|
+
nt_data = graph.serialize(format="nt")
|
|
182
|
+
|
|
183
|
+
dataset = pyoxigraph.Dataset(pyoxigraph.parse(nt_data, format=pyoxigraph.RdfFormat.N_TRIPLES))
|
|
184
|
+
dataset.canonicalize(pyoxigraph.CanonicalizationAlgorithm.RDFC_1_0)
|
|
185
|
+
|
|
186
|
+
canonical_quads = list(dataset)
|
|
187
|
+
|
|
188
|
+
# ── Phase 2: WL structural hashing for diff-stable blank node IDs
|
|
189
|
+
wl_map = _wl_signatures(canonical_quads)
|
|
190
|
+
|
|
191
|
+
def _remap(term):
|
|
192
|
+
if isinstance(term, pyoxigraph.BlankNode) and term.value in wl_map:
|
|
193
|
+
return pyoxigraph.BlankNode(wl_map[term.value])
|
|
194
|
+
return term
|
|
195
|
+
|
|
196
|
+
remapped = [pyoxigraph.Triple(_remap(q.subject), q.predicate, _remap(q.object)) for q in canonical_quads]
|
|
197
|
+
|
|
198
|
+
# ── Phase 3: Hybrid rdflib re-serialization ─────────────────────
|
|
199
|
+
# Convert pyoxigraph terms to rdflib terms and populate a clean
|
|
200
|
+
# Graph that only carries explicitly-bound prefixes.
|
|
201
|
+
def _to_rdflib(term):
|
|
202
|
+
"""Convert a pyoxigraph term to the equivalent rdflib term."""
|
|
203
|
+
if isinstance(term, pyoxigraph.NamedNode):
|
|
204
|
+
return URIRef(term.value)
|
|
205
|
+
if isinstance(term, pyoxigraph.BlankNode):
|
|
206
|
+
return BNode(term.value)
|
|
207
|
+
if isinstance(term, pyoxigraph.Literal):
|
|
208
|
+
if term.language:
|
|
209
|
+
return Literal(term.value, lang=term.language)
|
|
210
|
+
if term.datatype:
|
|
211
|
+
dt_iri = term.datatype.value
|
|
212
|
+
# In RDF 1.1, simple literals are syntactic sugar for
|
|
213
|
+
# xsd:string (Turtle §2.5.1). Preserve the shorter form
|
|
214
|
+
# to match the original owlgen output and avoid spurious
|
|
215
|
+
# diffs on every string literal.
|
|
216
|
+
if dt_iri == "http://www.w3.org/2001/XMLSchema#string":
|
|
217
|
+
return Literal(term.value)
|
|
218
|
+
return Literal(term.value, datatype=URIRef(dt_iri))
|
|
219
|
+
return Literal(term.value)
|
|
220
|
+
raise TypeError(f"Unexpected pyoxigraph term type: {type(term).__name__}: {term}")
|
|
221
|
+
|
|
222
|
+
result_graph = Graph(bind_namespaces="none")
|
|
223
|
+
for triple in remapped:
|
|
224
|
+
result_graph.add(
|
|
225
|
+
(
|
|
226
|
+
_to_rdflib(triple.subject),
|
|
227
|
+
_to_rdflib(triple.predicate),
|
|
228
|
+
_to_rdflib(triple.object),
|
|
229
|
+
)
|
|
230
|
+
)
|
|
231
|
+
|
|
232
|
+
# Bind only prefixes whose namespace IRI is actually referenced
|
|
233
|
+
# by at least one subject, predicate, or object in the graph.
|
|
234
|
+
# This filters out rdflib's ~27 built-in default bindings
|
|
235
|
+
# (brick, csvw, doap, …) that leak through Graph() even when
|
|
236
|
+
# the schema never declared them.
|
|
237
|
+
used_iris: set[str] = set()
|
|
238
|
+
for s, p, o in result_graph:
|
|
239
|
+
for term in (s, p, o):
|
|
240
|
+
if isinstance(term, URIRef):
|
|
241
|
+
used_iris.add(str(term))
|
|
242
|
+
|
|
243
|
+
for pfx, ns in sorted(graph.namespaces()):
|
|
244
|
+
pfx_s, ns_s = str(pfx), str(ns)
|
|
245
|
+
if pfx_s and any(iri.startswith(ns_s) for iri in used_iris):
|
|
246
|
+
result_graph.bind(pfx_s, ns_s)
|
|
247
|
+
|
|
248
|
+
# rdflib's Turtle serializer always emits a trailing double newline;
|
|
249
|
+
# normalize to a single newline for consistent file endings.
|
|
250
|
+
return result_graph.serialize(format="turtle").rstrip("\n") + "\n"
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def well_known_prefix_map() -> dict[str, str]:
|
|
256
|
+
"""Return a mapping from namespace URI to standard prefix name.
|
|
257
|
+
|
|
258
|
+
Uses rdflib's curated default namespace bindings as the source of truth.
|
|
259
|
+
For example, ``https://schema.org/`` maps to ``schema``.
|
|
260
|
+
|
|
261
|
+
This allows generators to normalise non-standard prefix aliases
|
|
262
|
+
(e.g. ``sdo`` for ``https://schema.org/``) to their conventional names.
|
|
263
|
+
"""
|
|
264
|
+
from rdflib import Graph as RdfGraph
|
|
265
|
+
|
|
266
|
+
return {str(ns): str(pfx) for pfx, ns in RdfGraph().namespaces() if str(pfx)}
|
|
267
|
+
|
|
268
|
+
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
"""Tests for the diffable-rdf public API."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
|
|
7
|
+
import pytest
|
|
8
|
+
from rdflib import BNode, Graph, Literal, Namespace, URIRef
|
|
9
|
+
from rdflib.compare import isomorphic
|
|
10
|
+
from rdflib.namespace import RDF, RDFS, XSD
|
|
11
|
+
|
|
12
|
+
from diffable_rdf import (
|
|
13
|
+
canonicalize_rdf_graph,
|
|
14
|
+
deterministic_json,
|
|
15
|
+
deterministic_turtle,
|
|
16
|
+
well_known_prefix_map,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
EX = Namespace("http://example.org/")
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _sample_graph() -> Graph:
|
|
23
|
+
g = Graph()
|
|
24
|
+
g.bind("ex", EX)
|
|
25
|
+
g.bind("rdfs", RDFS)
|
|
26
|
+
# a class with two restriction-like blank nodes
|
|
27
|
+
cls = EX.Thing
|
|
28
|
+
g.add((cls, RDF.type, RDFS.Class))
|
|
29
|
+
g.add((cls, RDFS.label, Literal("Thing", lang="en")))
|
|
30
|
+
g.add((cls, RDFS.comment, Literal("a plain string literal")))
|
|
31
|
+
for name, dt in (("width", XSD.double), ("count", XSD.integer)):
|
|
32
|
+
b = BNode()
|
|
33
|
+
g.add((cls, EX.constraint, b))
|
|
34
|
+
g.add((b, EX.onProperty, EX[name]))
|
|
35
|
+
g.add((b, EX.datatype, dt))
|
|
36
|
+
return g
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def test_deterministic_turtle_is_byte_stable_across_runs():
|
|
40
|
+
g = _sample_graph()
|
|
41
|
+
out1 = deterministic_turtle(g)
|
|
42
|
+
out2 = deterministic_turtle(_sample_graph())
|
|
43
|
+
assert out1 == out2
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def test_deterministic_turtle_is_isomorphic_to_input():
|
|
47
|
+
g = _sample_graph()
|
|
48
|
+
out = deterministic_turtle(g)
|
|
49
|
+
round_trip = Graph().parse(data=out, format="turtle")
|
|
50
|
+
assert isomorphic(round_trip, g)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def test_deterministic_turtle_strips_xsd_string():
|
|
54
|
+
g = Graph()
|
|
55
|
+
g.add((EX.s, EX.p, Literal("plain", datatype=XSD.string)))
|
|
56
|
+
out = deterministic_turtle(g)
|
|
57
|
+
assert "xsd:string" not in out
|
|
58
|
+
assert "^^" not in out
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def test_deterministic_turtle_uses_inline_blank_nodes():
|
|
62
|
+
g = _sample_graph()
|
|
63
|
+
out = deterministic_turtle(g)
|
|
64
|
+
assert "[" in out and "]" in out
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def test_deterministic_turtle_filters_unused_prefixes():
|
|
68
|
+
g = _sample_graph()
|
|
69
|
+
out = deterministic_turtle(g)
|
|
70
|
+
# rdflib binds ~27 default prefixes; only referenced ones should appear
|
|
71
|
+
assert "@prefix brick:" not in out
|
|
72
|
+
assert "@prefix ex:" in out
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def test_deterministic_turtle_no_bnodes():
|
|
76
|
+
g = Graph()
|
|
77
|
+
g.bind("ex", EX)
|
|
78
|
+
g.add((EX.a, EX.p, EX.b))
|
|
79
|
+
out1 = deterministic_turtle(g)
|
|
80
|
+
out2 = deterministic_turtle(g)
|
|
81
|
+
assert out1 == out2
|
|
82
|
+
assert isomorphic(Graph().parse(data=out1, format="turtle"), g)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def test_wl_stable_when_unrelated_triple_added():
|
|
86
|
+
g1 = _sample_graph()
|
|
87
|
+
g2 = _sample_graph()
|
|
88
|
+
g2.add((EX.Unrelated, RDF.type, RDFS.Class))
|
|
89
|
+
out1 = deterministic_turtle(g1)
|
|
90
|
+
out2 = deterministic_turtle(g2)
|
|
91
|
+
# the added statement must appear; the shared block's bnode labels
|
|
92
|
+
# (b<hash>) that survive should keep output overwhelmingly shared.
|
|
93
|
+
assert out1 != out2
|
|
94
|
+
shared = set(out1.splitlines()) & set(out2.splitlines())
|
|
95
|
+
assert len(shared) >= len(out1.splitlines()) // 2
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def test_deterministic_json_sorts_keys_and_lists():
|
|
99
|
+
obj = {"b": 1, "a": [3, 1, 2], "c": {"z": 0, "y": 1}}
|
|
100
|
+
out = deterministic_json(obj)
|
|
101
|
+
parsed = json.loads(out)
|
|
102
|
+
assert list(parsed.keys()) == ["a", "b", "c"]
|
|
103
|
+
assert parsed["a"] == [1, 2, 3]
|
|
104
|
+
assert list(parsed["c"].keys()) == ["y", "z"]
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def test_deterministic_json_preserves_context_order():
|
|
108
|
+
obj = {"@context": ["z", "a", "m"], "x": 1}
|
|
109
|
+
out = deterministic_json(obj)
|
|
110
|
+
parsed = json.loads(out)
|
|
111
|
+
assert parsed["@context"] == ["z", "a", "m"]
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def test_canonicalize_rdf_graph_is_deterministic():
|
|
115
|
+
g = _sample_graph()
|
|
116
|
+
out1 = canonicalize_rdf_graph(g, "turtle")
|
|
117
|
+
out2 = canonicalize_rdf_graph(_sample_graph(), "turtle")
|
|
118
|
+
assert out1 == out2
|
|
119
|
+
assert isomorphic(Graph().parse(data=out1, format="turtle"), g)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def test_canonicalize_rdf_graph_falls_back_for_literal_predicates():
|
|
123
|
+
# A literal in predicate position is non-standard RDF; must not crash.
|
|
124
|
+
g = Graph()
|
|
125
|
+
g.add((EX.s, EX.p, EX.o))
|
|
126
|
+
out = canonicalize_rdf_graph(g, "turtle")
|
|
127
|
+
assert "http://example.org/p" in out
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def test_well_known_prefix_map_contains_schema_org():
|
|
131
|
+
m = well_known_prefix_map()
|
|
132
|
+
assert m.get("https://schema.org/") == "schema"
|