arango-query-core 0.2.0__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- arango_query_core-0.2.2/.cursor/rules/fix-upstream-dont-fork.mdc +104 -0
- {arango_query_core-0.2.0 → arango_query_core-0.2.2}/CHANGELOG.md +47 -0
- {arango_query_core-0.2.0 → arango_query_core-0.2.2}/PKG-INFO +1 -1
- {arango_query_core-0.2.0 → arango_query_core-0.2.2}/arango_query_core/mapping.py +47 -2
- {arango_query_core-0.2.0 → arango_query_core-0.2.2}/arango_query_core/owl_rdflib.py +37 -7
- {arango_query_core-0.2.0 → arango_query_core-0.2.2}/arango_query_core/owl_turtle.py +41 -15
- {arango_query_core-0.2.0 → arango_query_core-0.2.2}/pyproject.toml +1 -1
- {arango_query_core-0.2.0 → arango_query_core-0.2.2}/tests/test_mapping_vci_detection.py +58 -0
- {arango_query_core-0.2.0 → arango_query_core-0.2.2}/tests/test_owl_rdflib.py +50 -0
- {arango_query_core-0.2.0 → arango_query_core-0.2.2}/tests/test_owl_turtle_roundtrip.py +88 -0
- arango_query_core-0.2.2/tests/test_upstream_markers.py +173 -0
- arango_query_core-0.2.0/.claude/settings.json +0 -8
- {arango_query_core-0.2.0 → arango_query_core-0.2.2}/.github/workflows/ci.yml +0 -0
- {arango_query_core-0.2.0 → arango_query_core-0.2.2}/.github/workflows/publish.yml +0 -0
- {arango_query_core-0.2.0 → arango_query_core-0.2.2}/.gitignore +0 -0
- {arango_query_core-0.2.0 → arango_query_core-0.2.2}/LICENSE +0 -0
- {arango_query_core-0.2.0 → arango_query_core-0.2.2}/README.md +0 -0
- {arango_query_core-0.2.0 → arango_query_core-0.2.2}/arango_query_core/__init__.py +0 -0
- {arango_query_core-0.2.0 → arango_query_core-0.2.2}/arango_query_core/aql.py +0 -0
- {arango_query_core-0.2.0 → arango_query_core-0.2.2}/arango_query_core/errors.py +0 -0
- {arango_query_core-0.2.0 → arango_query_core-0.2.2}/arango_query_core/exec.py +0 -0
- {arango_query_core-0.2.0 → arango_query_core-0.2.2}/arango_query_core/extensions.py +0 -0
- {arango_query_core-0.2.0 → arango_query_core-0.2.2}/arango_query_core/nl/__init__.py +0 -0
- {arango_query_core-0.2.0 → arango_query_core-0.2.2}/arango_query_core/nl/engine.py +0 -0
- {arango_query_core-0.2.0 → arango_query_core-0.2.2}/arango_query_core/nl/fewshot.py +0 -0
- {arango_query_core-0.2.0 → arango_query_core-0.2.2}/arango_query_core/nl/grounding.py +0 -0
- {arango_query_core-0.2.0 → arango_query_core-0.2.2}/arango_query_core/nl/pathindex.py +0 -0
- {arango_query_core-0.2.0 → arango_query_core-0.2.2}/arango_query_core/nl/postconditions.py +0 -0
- {arango_query_core-0.2.0 → arango_query_core-0.2.2}/arango_query_core/nl/providers.py +0 -0
- {arango_query_core-0.2.0 → arango_query_core-0.2.2}/arango_query_core/nl/seams.py +0 -0
- {arango_query_core-0.2.0 → arango_query_core-0.2.2}/arango_query_core/nl/synthbank.py +0 -0
- {arango_query_core-0.2.0 → arango_query_core-0.2.2}/arango_query_core/py.typed +0 -0
- {arango_query_core-0.2.0 → arango_query_core-0.2.2}/docs/PRD.md +0 -0
- {arango_query_core-0.2.0 → arango_query_core-0.2.2}/tests/fixtures/mappings/pg.owl.ttl +0 -0
- {arango_query_core-0.2.0 → arango_query_core-0.2.2}/tests/test_mapping_helpers.py +0 -0
- {arango_query_core-0.2.0 → arango_query_core-0.2.2}/tests/test_mapping_resolver_normalization.py +0 -0
- {arango_query_core-0.2.0 → arango_query_core-0.2.2}/tests/test_nl_engine.py +0 -0
- {arango_query_core-0.2.0 → arango_query_core-0.2.2}/tests/test_nl_fewshot.py +0 -0
- {arango_query_core-0.2.0 → arango_query_core-0.2.2}/tests/test_nl_grounding.py +0 -0
- {arango_query_core-0.2.0 → arango_query_core-0.2.2}/tests/test_nl_pathindex.py +0 -0
- {arango_query_core-0.2.0 → arango_query_core-0.2.2}/tests/test_nl_postconditions.py +0 -0
- {arango_query_core-0.2.0 → arango_query_core-0.2.2}/tests/test_nl_providers.py +0 -0
- {arango_query_core-0.2.0 → arango_query_core-0.2.2}/tests/test_nl_synthbank.py +0 -0
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
---
|
|
2
|
+
description: When a library we own lacks something or gets it wrong, fix it in the library or file it there; a workaround in the consumer is temporary, marked and tracked
|
|
3
|
+
alwaysApply: true
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Fix Upstream, Don't Fork
|
|
7
|
+
|
|
8
|
+
> **Scope:** Any code that depends on a library we own — arango-sparql-py,
|
|
9
|
+
> arango-query-core, r2g / r2g-arango, arango-schema-analyzer, arango-cypher-py,
|
|
10
|
+
> arango-entity-resolution, or any other library the team maintains. Every one of
|
|
11
|
+
> these is also a consumer of the others, so this applies inside the libraries too.
|
|
12
|
+
|
|
13
|
+
> **"We own the library. A gap in it is an issue there, not a fork here."**
|
|
14
|
+
|
|
15
|
+
Written after a 2026-10-15 finding in contextual-data-fabric: its English→SPARQL
|
|
16
|
+
translator was never the library's. In July, arango-sparql-py could only check a
|
|
17
|
+
generated query against one schema, so the consumer took the library's model client
|
|
18
|
+
and wrote its own prompt, check and repair loop around it. That was a reasonable
|
|
19
|
+
call — but no library issue was filed and nothing in the code said it was
|
|
20
|
+
temporary. When the shared engine gained the seam the consumer needed, nobody
|
|
21
|
+
noticed. Three months later there were three implementations, the design record
|
|
22
|
+
still said the engine was reused, and the library's docstring named a function the
|
|
23
|
+
consumer never imported.
|
|
24
|
+
|
|
25
|
+
The same audit found the pattern again around arango-entity-resolution: a consumer
|
|
26
|
+
guard that copies the resolver's checks line for line, configuration defects worked
|
|
27
|
+
around in a comment instead of filed, and a service that declares the library as a
|
|
28
|
+
dependency, says in its docstring that it runs the library's pipeline, and imports
|
|
29
|
+
none of it.
|
|
30
|
+
|
|
31
|
+
## When an owned library falls short
|
|
32
|
+
|
|
33
|
+
| Do, in this order | Not |
|
|
34
|
+
| --- | --- |
|
|
35
|
+
| 1. **Check it isn't already there** — `main`, open branches, and the other org's copy | Assume the version you have pinned is the latest |
|
|
36
|
+
| 2. **Fix it in the library** — we own it; open the pull request there | Patch around it here because this is the repo you have open |
|
|
37
|
+
| 3. **Or file it there** — a bug or feature request on the repo that leads, saying what the consumer needs and why | File it on the mirror, or mention it only in this repo's pull request |
|
|
38
|
+
| 4. **Only if this repo cannot wait** for a library release and a pin bump — a temporary workaround, held to the rules below | A permanent workaround nobody can find |
|
|
39
|
+
|
|
40
|
+
**Which repo leads.** Most of these libraries exist under two owners, and which one
|
|
41
|
+
leads has changed during 2026. Do not trust the remotes of your clone — a fresh
|
|
42
|
+
clone has only the URL you were handed. The repository with recent pull requests is
|
|
43
|
+
the one that leads:
|
|
44
|
+
|
|
45
|
+
```sh
|
|
46
|
+
gh pr list --repo <owner>/<name> --state all --limit 5
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
Run it against both candidates. If neither has recent pull requests, ask.
|
|
50
|
+
|
|
51
|
+
## A workaround that is allowed
|
|
52
|
+
|
|
53
|
+
- **Behind the seam the library would provide**, so removing it touches one place:
|
|
54
|
+
an adapter, or a wrapper with the library's own shape.
|
|
55
|
+
- **Marked where it lives**, on one line at the code that works around the library:
|
|
56
|
+
|
|
57
|
+
```python
|
|
58
|
+
# upstream: <org>/arango-sparql-py#<n> — remove when the engine accepts a caller's validator
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
Where a repo has `tests/test_upstream_markers.py`, that test holds the format: an
|
|
62
|
+
`owner/repo#number` issue a person can open, and a `remove when` condition someone
|
|
63
|
+
can check. The shared-memory installers place this rule but not the test, so a
|
|
64
|
+
repo without it copies it from arango-shared-memory
|
|
65
|
+
`templates/tests/test_upstream_markers.py`.
|
|
66
|
+
- **Listed in this repo's known-debt issue**, if it keeps one, with the same link.
|
|
67
|
+
- **Removed in the pin bump that ships the fix**, not in a later cleanup: when you
|
|
68
|
+
bump a pin, search for `upstream:` markers that name that library.
|
|
69
|
+
|
|
70
|
+
## What counts as a fork
|
|
71
|
+
|
|
72
|
+
| Fork | Why it costs |
|
|
73
|
+
| --- | --- |
|
|
74
|
+
| Taking a library's low-level piece (a client, a parser) and writing your own version of its main entry point around it | Fixes and new capabilities land in the library's version and never reach yours |
|
|
75
|
+
| Copying library code into the consumer | The two copies drift |
|
|
76
|
+
| Patching the library at runtime | The next pin bump breaks it without a word |
|
|
77
|
+
| Reaching past the library's public API into its internals | Internals move without a release note |
|
|
78
|
+
| Defining a public function or class with the same name and job as one the library exports | Readers take one for the other |
|
|
79
|
+
| Declaring a library as a dependency and reimplementing what you would have called | The dependency says one thing and the code does another |
|
|
80
|
+
| Holding a pin back to dodge a bug, with no issue | Nobody knows when it can move |
|
|
81
|
+
|
|
82
|
+
A fork that is deliberate and permanent — the consumer needs something the library
|
|
83
|
+
should not hold — is a **design record (ADR) in the consumer** plus an **issue in the
|
|
84
|
+
library** for the seam that would let the two share the rest. It is never a silent
|
|
85
|
+
decision.
|
|
86
|
+
|
|
87
|
+
## The other side: claims about the other repo
|
|
88
|
+
|
|
89
|
+
A library that says a consumer uses it ("this is what the fabric calls") must be able
|
|
90
|
+
to point at the consumer's import. A consumer that says it uses a library ("runs the
|
|
91
|
+
entity-resolution pipeline") must import it. If either cannot, the claim is a plan,
|
|
92
|
+
and it is written as one.
|
|
93
|
+
|
|
94
|
+
## Anti-Patterns
|
|
95
|
+
|
|
96
|
+
| Anti-Pattern | Consequence |
|
|
97
|
+
| --- | --- |
|
|
98
|
+
| "The library can't do X, so I'll write X here" | A fork nobody tracks |
|
|
99
|
+
| A workaround with no `upstream:` marker | Nobody removes it when the fix ships |
|
|
100
|
+
| An issue filed in the consumer instead of the library | The library's maintainers never see it |
|
|
101
|
+
| A library defect documented only in a config comment | The next consumer hits it too |
|
|
102
|
+
| "It's our library, I'll fix it there later" | Later is after the third implementation |
|
|
103
|
+
|
|
104
|
+
**We own the library: fix it there, or say where it will be fixed.**
|
|
@@ -6,6 +6,53 @@ exact versions (contextual-data-fabric CC-9) and re-run their golden /
|
|
|
6
6
|
eval suites on every bump — pre-1.0, **minor** bumps may break the API
|
|
7
7
|
and this file must say so explicitly.
|
|
8
8
|
|
|
9
|
+
## [0.2.2] - 2026-10-21
|
|
10
|
+
|
|
11
|
+
### Fixed
|
|
12
|
+
|
|
13
|
+
- **Traversals no longer hint an array-expanded edge index.** `resolve_indexes`
|
|
14
|
+
treated any edge index led by `_from` / `_to` as a vertex-centric index, so
|
|
15
|
+
`[_from, type, toLabels[*]]` (r2g's opt-in `LpgLayout.index_edge_labels`
|
|
16
|
+
shape) could become arango-cypher-py's traversal `indexHint`. A persistent
|
|
17
|
+
index stores one entry per array element and a traversal scans all of them,
|
|
18
|
+
so the hinted traversal returned each edge once per label -- inflated counts,
|
|
19
|
+
no error (measured on ArangoDB 3.12.9: 20x the rows at one hop, 400x at two).
|
|
20
|
+
Such an index is now never `vci`, even with an explicit `vci: true` flag. With
|
|
21
|
+
only that index, arango-cypher-py reports a missing VCI instead of hinting it;
|
|
22
|
+
next to a flat `[_from, type]` index, the flat one is hinted, whatever the
|
|
23
|
+
order.
|
|
24
|
+
- **The schema analyzer's verdict is honored.** When a relationship mapping
|
|
25
|
+
carries arangodb-schema-analyzer's `vci.indexLevel.indexes[].traversalSafe:
|
|
26
|
+
false` (analyzer 0.15.2+), that index is not traversal-safe, whatever its
|
|
27
|
+
shape. Older exports carry no verdict; the local array-expansion check still
|
|
28
|
+
applies to them.
|
|
29
|
+
|
|
30
|
+
### Added
|
|
31
|
+
|
|
32
|
+
- `IndexInfo.traversal_safe` (default `True`): False when a traversal must not
|
|
33
|
+
use the index. Additive; existing callers are unaffected.
|
|
34
|
+
|
|
35
|
+
## [0.2.1] - 2026-10-16
|
|
36
|
+
|
|
37
|
+
### Fixed
|
|
38
|
+
|
|
39
|
+
- **OWL import keeps property names.** `owl_turtle.turtle_to_mapping` cut
|
|
40
|
+
every datatype-property name at its last underscore — a rule meant for the
|
|
41
|
+
`:<REL>_<prop>` relationship properties `mapping_to_turtle` writes — so
|
|
42
|
+
`file_name` imported as `name` and `account_id` as `id`, including on a
|
|
43
|
+
round trip of this library's own export. Entity properties now keep their
|
|
44
|
+
full name; relationship properties lose exactly their `<REL>_` prefix.
|
|
45
|
+
- **Shared fields reach every class.** Both importers (`turtle_to_mapping`
|
|
46
|
+
and the rdflib-based `parse_owl_with_rdflib`) dropped a property whose
|
|
47
|
+
`rdfs:domain` is an `owl:unionOf` list — arangodb-schema-analyzer
|
|
48
|
+
0.15.1's form for a field several classes share — and the rdflib one read
|
|
49
|
+
only the first of several domains. Each member / domain now gets the field,
|
|
50
|
+
and the anonymous union class is no longer imported as an entity.
|
|
51
|
+
- **Stored field names.** `phys:attributeName` (analyzer 0.15.1) now maps a
|
|
52
|
+
property to its stored field in the physical mapping
|
|
53
|
+
(`properties.<name>.field`), so a sanitized or renamed field (`file-name`,
|
|
54
|
+
`page_count`) resolves to the attribute that holds it.
|
|
55
|
+
|
|
9
56
|
## [0.2.0] - 2026-08-23
|
|
10
57
|
|
|
11
58
|
### BREAKING
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: arango-query-core
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: Shared conceptual-model and NL substrate for ArangoDB query transpilers (arango-cypher-py, arango-sparql-py)
|
|
5
5
|
Project-URL: Repository, https://github.com/ArthurKeen/arango-query-core
|
|
6
6
|
Project-URL: Changelog, https://github.com/ArthurKeen/arango-query-core/blob/main/CHANGELOG.md
|
|
@@ -87,9 +87,49 @@ def _is_structural_vci(index_type: str, fields: tuple[str, ...]) -> bool:
|
|
|
87
87
|
return any(f not in _SYSTEM_INDEX_FIELDS for f in fields)
|
|
88
88
|
|
|
89
89
|
|
|
90
|
+
def _has_array_expansion(fields: tuple[str, ...]) -> bool:
|
|
91
|
+
"""True when any indexed field is an array expansion (``labels[*]``).
|
|
92
|
+
|
|
93
|
+
A persistent index stores one entry per array element, and a traversal
|
|
94
|
+
scans every entry under the ``_from`` / ``_to`` prefix, so a traversal that
|
|
95
|
+
uses such an index returns each edge once per element -- inflated counts,
|
|
96
|
+
no error. The index is still correct for a pattern-match lookup
|
|
97
|
+
(``FOR e IN edges FILTER e._from == ...``), but never for a traversal hint.
|
|
98
|
+
"""
|
|
99
|
+
return any("[*]" in f for f in fields)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _analyzer_unsafe_index_fields(mapping: JsonObj) -> set[tuple[str, ...]]:
|
|
103
|
+
"""Field tuples the schema analyzer marked ``traversalSafe: false``.
|
|
104
|
+
|
|
105
|
+
arangodb-schema-analyzer (0.15.2+) records its verdict per relationship
|
|
106
|
+
under ``vci.indexLevel.indexes``; ``traversalSafe`` is present only when
|
|
107
|
+
false. Older exports carry no verdict, so absence means "no opinion" and
|
|
108
|
+
the local :func:`_has_array_expansion` check still applies.
|
|
109
|
+
"""
|
|
110
|
+
vci = mapping.get("vci")
|
|
111
|
+
index_level = vci.get("indexLevel") if isinstance(vci, dict) else None
|
|
112
|
+
indexes = index_level.get("indexes") if isinstance(index_level, dict) else None
|
|
113
|
+
if not isinstance(indexes, list):
|
|
114
|
+
return set()
|
|
115
|
+
unsafe: set[tuple[str, ...]] = set()
|
|
116
|
+
for entry in indexes:
|
|
117
|
+
if not isinstance(entry, dict) or entry.get("traversalSafe") is not False:
|
|
118
|
+
continue
|
|
119
|
+
fields = entry.get("fields")
|
|
120
|
+
if isinstance(fields, list):
|
|
121
|
+
unsafe.add(tuple(str(f) for f in fields))
|
|
122
|
+
return unsafe
|
|
123
|
+
|
|
124
|
+
|
|
90
125
|
@dataclass(frozen=True)
|
|
91
126
|
class IndexInfo:
|
|
92
|
-
"""Metadata for a single index on a collection.
|
|
127
|
+
"""Metadata for a single index on a collection.
|
|
128
|
+
|
|
129
|
+
``traversal_safe`` is False when a traversal must not use the index
|
|
130
|
+
(array-expanded fields, or the schema analyzer's ``traversalSafe: false``
|
|
131
|
+
verdict). Such an index is never ``vci``, whatever its shape or flags.
|
|
132
|
+
"""
|
|
93
133
|
|
|
94
134
|
type: str
|
|
95
135
|
fields: tuple[str, ...]
|
|
@@ -98,6 +138,7 @@ class IndexInfo:
|
|
|
98
138
|
name: str = ""
|
|
99
139
|
vci: bool = False
|
|
100
140
|
deduplicate: bool = False
|
|
141
|
+
traversal_safe: bool = True
|
|
101
142
|
|
|
102
143
|
|
|
103
144
|
@dataclass(frozen=True)
|
|
@@ -573,6 +614,7 @@ class MappingResolver:
|
|
|
573
614
|
if not isinstance(indexes_raw, list):
|
|
574
615
|
return []
|
|
575
616
|
|
|
617
|
+
unsafe = _analyzer_unsafe_index_fields(mapping)
|
|
576
618
|
result: list[IndexInfo] = []
|
|
577
619
|
for idx in indexes_raw:
|
|
578
620
|
if not isinstance(idx, dict):
|
|
@@ -583,6 +625,8 @@ class MappingResolver:
|
|
|
583
625
|
else:
|
|
584
626
|
continue
|
|
585
627
|
index_type = str(idx.get("type", "persistent"))
|
|
628
|
+
traversal_safe = fields not in unsafe and not _has_array_expansion(fields)
|
|
629
|
+
vci = bool(idx.get("vci", False)) or _is_structural_vci(index_type, fields)
|
|
586
630
|
result.append(
|
|
587
631
|
IndexInfo(
|
|
588
632
|
type=index_type,
|
|
@@ -590,8 +634,9 @@ class MappingResolver:
|
|
|
590
634
|
unique=bool(idx.get("unique", False)),
|
|
591
635
|
sparse=bool(idx.get("sparse", False)),
|
|
592
636
|
name=str(idx.get("name", "")),
|
|
593
|
-
vci=
|
|
637
|
+
vci=vci and traversal_safe,
|
|
594
638
|
deduplicate=bool(idx.get("deduplicate", False)),
|
|
639
|
+
traversal_safe=traversal_safe,
|
|
595
640
|
)
|
|
596
641
|
)
|
|
597
642
|
return result
|
|
@@ -42,23 +42,30 @@ def parse_owl_with_rdflib(turtle_text: str) -> MappingBundle:
|
|
|
42
42
|
entities: list[dict[str, Any]] = []
|
|
43
43
|
entity_props: dict[str, list[dict[str, str]]] = {}
|
|
44
44
|
|
|
45
|
-
for cls in g
|
|
45
|
+
for cls in _named_classes(g, RDF, OWL):
|
|
46
46
|
name = _local_name(cls)
|
|
47
47
|
entities.append({"name": name, "labels": [name], "properties": []})
|
|
48
48
|
entity_props[name] = []
|
|
49
49
|
|
|
50
50
|
# ── Extract datatype properties ──────────────────────────────────
|
|
51
|
+
# A property's domain may be one class, several (each gets the field),
|
|
52
|
+
# or an owl:unionOf class expression — arangodb-schema-analyzer's form
|
|
53
|
+
# for a field several classes share. phys:attributeName names the
|
|
54
|
+
# stored field when it differs from the property name.
|
|
55
|
+
fields: dict[str, dict[str, str]] = {}
|
|
51
56
|
for prop in g.subjects(RDF.type, OWL.DatatypeProperty):
|
|
52
57
|
prop_name = _local_name(prop)
|
|
53
|
-
domain = g.value(prop, RDFS.domain)
|
|
54
58
|
range_uri = g.value(prop, RDFS.range)
|
|
55
59
|
prop_entry: dict[str, str] = {"name": prop_name}
|
|
56
60
|
if range_uri:
|
|
57
61
|
prop_entry["type"] = _xsd_to_simple(str(range_uri))
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
entity_props
|
|
62
|
+
attribute = g.value(prop, PHYS.attributeName)
|
|
63
|
+
for domain_node in g.objects(prop, RDFS.domain):
|
|
64
|
+
for domain_name in _domain_names(g, domain_node, OWL):
|
|
65
|
+
if domain_name in entity_props:
|
|
66
|
+
entity_props[domain_name].append(prop_entry)
|
|
67
|
+
if attribute:
|
|
68
|
+
fields.setdefault(domain_name, {})[prop_name] = str(attribute)
|
|
62
69
|
|
|
63
70
|
for entity in entities:
|
|
64
71
|
entity["properties"] = entity_props.get(entity["name"], [])
|
|
@@ -82,7 +89,7 @@ def parse_owl_with_rdflib(turtle_text: str) -> MappingBundle:
|
|
|
82
89
|
physical_entities: dict[str, dict[str, Any]] = {}
|
|
83
90
|
physical_rels: dict[str, dict[str, Any]] = {}
|
|
84
91
|
|
|
85
|
-
for cls in g
|
|
92
|
+
for cls in _named_classes(g, RDF, OWL):
|
|
86
93
|
name = _local_name(cls)
|
|
87
94
|
coll = g.value(cls, PHYS.collectionName)
|
|
88
95
|
style = g.value(cls, PHYS.style) or g.value(cls, PHYS.mappingStyle)
|
|
@@ -98,6 +105,8 @@ def parse_owl_with_rdflib(turtle_text: str) -> MappingBundle:
|
|
|
98
105
|
pm_entry["typeField"] = str(type_field)
|
|
99
106
|
if type_value:
|
|
100
107
|
pm_entry["typeValue"] = str(type_value)
|
|
108
|
+
for prop_name, field in fields.get(name, {}).items():
|
|
109
|
+
pm_entry["properties"][prop_name] = {"field": field}
|
|
101
110
|
physical_entities[name] = pm_entry
|
|
102
111
|
|
|
103
112
|
for prop in g.subjects(RDF.type, OWL.ObjectProperty):
|
|
@@ -127,6 +136,27 @@ def parse_owl_with_rdflib(turtle_text: str) -> MappingBundle:
|
|
|
127
136
|
)
|
|
128
137
|
|
|
129
138
|
|
|
139
|
+
def _named_classes(g: Any, rdf: Any, owl: Any) -> list[Any]:
|
|
140
|
+
"""Every ``owl:Class`` with an IRI. An anonymous class expression (the
|
|
141
|
+
``[ a owl:Class ; owl:unionOf ( … ) ]`` of a shared field's domain) is not
|
|
142
|
+
an entity."""
|
|
143
|
+
import rdflib
|
|
144
|
+
|
|
145
|
+
return [c for c in g.subjects(rdf.type, owl.Class) if isinstance(c, rdflib.URIRef)]
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _domain_names(g: Any, domain: Any, owl: Any) -> list[str]:
|
|
149
|
+
"""Class names a ``rdfs:domain`` value covers: the class itself, or every
|
|
150
|
+
member of an ``owl:unionOf`` list."""
|
|
151
|
+
import rdflib
|
|
152
|
+
from rdflib.collection import Collection
|
|
153
|
+
|
|
154
|
+
if isinstance(domain, rdflib.BNode):
|
|
155
|
+
members = g.value(domain, owl.unionOf)
|
|
156
|
+
return [_local_name(m) for m in Collection(g, members)] if members is not None else []
|
|
157
|
+
return [_local_name(domain)]
|
|
158
|
+
|
|
159
|
+
|
|
130
160
|
def _local_name(uri: Any) -> str:
|
|
131
161
|
"""Extract the local (fragment / last-path-segment) name from a URI."""
|
|
132
162
|
s = str(uri)
|
|
@@ -111,10 +111,21 @@ def mapping_to_turtle(bundle: MappingBundle) -> str:
|
|
|
111
111
|
|
|
112
112
|
|
|
113
113
|
def turtle_to_mapping(turtle: str) -> MappingBundle:
|
|
114
|
-
"""Parse an OWL/Turtle string into a MappingBundle.
|
|
114
|
+
"""Parse an OWL/Turtle string into a MappingBundle.
|
|
115
|
+
|
|
116
|
+
A line-oriented reader for the layout :func:`mapping_to_turtle` and
|
|
117
|
+
arangodb-schema-analyzer write (no rdflib needed; see
|
|
118
|
+
:mod:`.owl_rdflib` for arbitrary Turtle). A datatype property keeps its
|
|
119
|
+
full local name (``:file_name`` is ``file_name``); only a relationship
|
|
120
|
+
property written ``:<REL>_<prop>`` with ``rdfs:domain :<REL>`` loses the
|
|
121
|
+
``<REL>_`` prefix. Its domain may be one class, several ``rdfs:domain``
|
|
122
|
+
lines, or an ``owl:unionOf`` list (the analyzer's form for a field
|
|
123
|
+
several classes share), and ``phys:attributeName`` names the stored
|
|
124
|
+
field when it differs from the property name.
|
|
125
|
+
"""
|
|
115
126
|
classes: dict[str, dict[str, str]] = {}
|
|
116
127
|
obj_props: dict[str, dict[str, str]] = {}
|
|
117
|
-
data_props: list[dict[str,
|
|
128
|
+
data_props: list[dict[str, Any]] = []
|
|
118
129
|
annotations: dict[str, dict[str, str]] = {}
|
|
119
130
|
|
|
120
131
|
current_subject: str | None = None
|
|
@@ -143,14 +154,20 @@ def turtle_to_mapping(turtle: str) -> MappingBundle:
|
|
|
143
154
|
if m_dataprop:
|
|
144
155
|
current_subject = m_dataprop.group(1)
|
|
145
156
|
current_kind = "dataprop"
|
|
146
|
-
data_props.append({"name": current_subject})
|
|
157
|
+
data_props.append({"name": current_subject, "domains": []})
|
|
158
|
+
continue
|
|
159
|
+
|
|
160
|
+
m_union = re.match(r"rdfs:domain\s+\[.*owl:unionOf\s*\(([^)]*)\)", stripped)
|
|
161
|
+
if m_union:
|
|
162
|
+
if current_kind == "dataprop" and data_props:
|
|
163
|
+
data_props[-1]["domains"].extend(re.findall(r":(\w+)", m_union.group(1)))
|
|
147
164
|
continue
|
|
148
165
|
|
|
149
166
|
m_domain = re.match(r"rdfs:domain\s+:(\w+)", stripped)
|
|
150
167
|
if m_domain:
|
|
151
168
|
domain = m_domain.group(1)
|
|
152
169
|
if current_kind == "dataprop" and data_props:
|
|
153
|
-
data_props[-1]["
|
|
170
|
+
data_props[-1]["domains"].append(domain)
|
|
154
171
|
elif current_kind == "objprop" and current_subject:
|
|
155
172
|
obj_props[current_subject]["domain"] = domain
|
|
156
173
|
continue
|
|
@@ -164,7 +181,14 @@ def turtle_to_mapping(turtle: str) -> MappingBundle:
|
|
|
164
181
|
obj_props[current_subject]["range"] = rng.lstrip(":")
|
|
165
182
|
continue
|
|
166
183
|
|
|
167
|
-
for annot in (
|
|
184
|
+
for annot in (
|
|
185
|
+
"mappingStyle",
|
|
186
|
+
"collectionName",
|
|
187
|
+
"typeField",
|
|
188
|
+
"typeValue",
|
|
189
|
+
"edgeCollectionName",
|
|
190
|
+
"attributeName",
|
|
191
|
+
):
|
|
168
192
|
m_a = re.match(rf':(\w+)\s+phys:{annot}\s+"([^"]+)"', stripped)
|
|
169
193
|
if m_a:
|
|
170
194
|
subject = m_a.group(1)
|
|
@@ -174,14 +198,8 @@ def turtle_to_mapping(turtle: str) -> MappingBundle:
|
|
|
174
198
|
entities_cs: list[dict[str, Any]] = []
|
|
175
199
|
entities_pm: dict[str, dict[str, Any]] = {}
|
|
176
200
|
for name in classes:
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
"name": dp["name"].split("_")[-1] if "_" in dp["name"] else dp["name"],
|
|
180
|
-
"type": _from_xsd_type(dp.get("range", "xsd:string")),
|
|
181
|
-
}
|
|
182
|
-
for dp in data_props
|
|
183
|
-
if dp.get("domain") == name
|
|
184
|
-
]
|
|
201
|
+
own = [dp for dp in data_props if name in dp["domains"]]
|
|
202
|
+
props = [{"name": dp["name"], "type": _from_xsd_type(dp.get("range", "xsd:string"))} for dp in own]
|
|
185
203
|
entities_cs.append({"name": name, "labels": [name], "properties": props})
|
|
186
204
|
ann = annotations.get(name, {})
|
|
187
205
|
pm_entry: dict[str, Any] = {
|
|
@@ -191,6 +209,13 @@ def turtle_to_mapping(turtle: str) -> MappingBundle:
|
|
|
191
209
|
if pm_entry["style"] == "LABEL":
|
|
192
210
|
pm_entry["typeField"] = ann.get("typeField", "type")
|
|
193
211
|
pm_entry["typeValue"] = ann.get("typeValue", name)
|
|
212
|
+
fields = {
|
|
213
|
+
dp["name"]: {"field": annotations[dp["name"]]["attributeName"]}
|
|
214
|
+
for dp in own
|
|
215
|
+
if annotations.get(dp["name"], {}).get("attributeName")
|
|
216
|
+
}
|
|
217
|
+
if fields:
|
|
218
|
+
pm_entry["properties"] = fields
|
|
194
219
|
entities_pm[name] = pm_entry
|
|
195
220
|
|
|
196
221
|
rels_cs: list[dict[str, Any]] = []
|
|
@@ -201,11 +226,12 @@ def turtle_to_mapping(turtle: str) -> MappingBundle:
|
|
|
201
226
|
to_e = op.get("range", "Any")
|
|
202
227
|
props = [
|
|
203
228
|
{
|
|
204
|
-
|
|
229
|
+
# Exactly the `<REL>_` prefix mapping_to_turtle adds, not every occurrence.
|
|
230
|
+
"name": dp["name"].removeprefix(f"{rtype}_"),
|
|
205
231
|
"type": _from_xsd_type(dp.get("range", "xsd:string")),
|
|
206
232
|
}
|
|
207
233
|
for dp in data_props
|
|
208
|
-
if dp
|
|
234
|
+
if rtype in dp["domains"]
|
|
209
235
|
]
|
|
210
236
|
rels_cs.append({"type": rtype, "fromEntity": from_e, "toEntity": to_e, "properties": props})
|
|
211
237
|
ann = annotations.get(rtype, {})
|
|
@@ -86,3 +86,61 @@ class TestResolverStructuralVci:
|
|
|
86
86
|
def test_no_indexes_means_no_vci(self) -> None:
|
|
87
87
|
b = _bundle_with_edge_indexes([])
|
|
88
88
|
assert MappingResolver(b).has_vci("REL") is False
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
class TestArrayExpandedIndexIsNotTraversalSafe:
|
|
92
|
+
"""An edge index with an array-expanded field must never be hinted in a traversal.
|
|
93
|
+
|
|
94
|
+
A persistent index stores one entry per array element and a traversal scans
|
|
95
|
+
every entry under the _from/_to prefix, so ``[_from, type, toLabels[*]]``
|
|
96
|
+
returns each edge once per label (measured on ArangoDB 3.12.9: 20 labels
|
|
97
|
+
per edge -> 20x the rows at one hop, 400x at two). r2g provisions this shape
|
|
98
|
+
when ``LpgLayout.index_edge_labels`` is on.
|
|
99
|
+
"""
|
|
100
|
+
|
|
101
|
+
def test_array_expanded_index_is_not_vci(self) -> None:
|
|
102
|
+
b = _bundle_with_edge_indexes(
|
|
103
|
+
[{"type": "persistent", "fields": ["_from", "type", "toLabels[*]"], "name": "lbl"}]
|
|
104
|
+
)
|
|
105
|
+
r = MappingResolver(b)
|
|
106
|
+
(idx,) = r.resolve_indexes("REL")
|
|
107
|
+
assert idx.traversal_safe is False
|
|
108
|
+
assert idx.vci is False
|
|
109
|
+
assert r.has_vci("REL") is False
|
|
110
|
+
|
|
111
|
+
def test_explicit_flag_does_not_override_array_expansion(self) -> None:
|
|
112
|
+
b = _bundle_with_edge_indexes(
|
|
113
|
+
[{"type": "persistent", "fields": ["_from", "toLabels[*]"], "name": "lbl", "vci": True}]
|
|
114
|
+
)
|
|
115
|
+
assert MappingResolver(b).has_vci("REL") is False
|
|
116
|
+
|
|
117
|
+
def test_safe_index_still_vci_alongside_risky_one(self) -> None:
|
|
118
|
+
b = _bundle_with_edge_indexes(
|
|
119
|
+
[
|
|
120
|
+
{"type": "persistent", "fields": ["_from", "type"], "name": "flat"},
|
|
121
|
+
{"type": "persistent", "fields": ["_from", "type", "toLabels[*]"], "name": "lbl"},
|
|
122
|
+
]
|
|
123
|
+
)
|
|
124
|
+
by_name = {i.name: i for i in MappingResolver(b).resolve_indexes("REL")}
|
|
125
|
+
assert by_name["flat"].vci is True and by_name["flat"].traversal_safe is True
|
|
126
|
+
assert by_name["lbl"].vci is False
|
|
127
|
+
|
|
128
|
+
def test_analyzer_verdict_is_honored(self) -> None:
|
|
129
|
+
# The schema analyzer's traversalSafe: false wins even for a shape the
|
|
130
|
+
# local check would accept -- its verdict is the source of truth.
|
|
131
|
+
b = _bundle_with_edge_indexes([{"type": "persistent", "fields": ["_from", "type"], "name": "flat"}])
|
|
132
|
+
b.physical_mapping["relationships"]["REL"]["vci"] = {
|
|
133
|
+
"indexLevel": {"indexes": [{"fields": ["_from", "type"], "traversalSafe": False}]}
|
|
134
|
+
}
|
|
135
|
+
(idx,) = MappingResolver(b).resolve_indexes("REL")
|
|
136
|
+
assert idx.traversal_safe is False and idx.vci is False
|
|
137
|
+
|
|
138
|
+
def test_analyzer_block_without_verdict_keeps_safe_index(self) -> None:
|
|
139
|
+
# Exports before analyzer 0.15.2 have a vci block but no traversalSafe:
|
|
140
|
+
# absence is "no opinion", not "unsafe".
|
|
141
|
+
b = _bundle_with_edge_indexes([{"type": "persistent", "fields": ["_from", "type"], "name": "flat"}])
|
|
142
|
+
b.physical_mapping["relationships"]["REL"]["vci"] = {
|
|
143
|
+
"indexLevel": {"indexes": [{"fields": ["_from", "type"], "discriminatorFields": ["type"]}]}
|
|
144
|
+
}
|
|
145
|
+
(idx,) = MappingResolver(b).resolve_indexes("REL")
|
|
146
|
+
assert idx.traversal_safe is True and idx.vci is True
|
|
@@ -189,3 +189,53 @@ class TestImportError:
|
|
|
189
189
|
with patch("builtins.__import__", side_effect=mock_import):
|
|
190
190
|
with pytest.raises(ImportError, match="rdflib is required"):
|
|
191
191
|
parse_owl_with_rdflib(MINIMAL_OWL)
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
ANALYZER_OWL = """\
|
|
195
|
+
@prefix : <http://arangodb.com/schema/hybrid#> .
|
|
196
|
+
@prefix owl: <http://www.w3.org/2002/07/owl#> .
|
|
197
|
+
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
|
|
198
|
+
@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
|
|
199
|
+
@prefix phys: <http://arangodb.com/schema/physical#> .
|
|
200
|
+
|
|
201
|
+
:Chunk a owl:Class ; phys:collectionName "chunks" .
|
|
202
|
+
:Document a owl:Class ; phys:collectionName "documents" .
|
|
203
|
+
:Note a owl:Class ; phys:collectionName "notes" .
|
|
204
|
+
|
|
205
|
+
:content a owl:DatatypeProperty ;
|
|
206
|
+
rdfs:domain [ a owl:Class ; owl:unionOf ( :Chunk :Document ) ] .
|
|
207
|
+
:file_name a owl:DatatypeProperty ;
|
|
208
|
+
rdfs:domain :Document ;
|
|
209
|
+
phys:attributeName "file-name" .
|
|
210
|
+
:tag a owl:DatatypeProperty ;
|
|
211
|
+
rdfs:domain :Document , :Note ;
|
|
212
|
+
rdfs:range xsd:string .
|
|
213
|
+
"""
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def _names(bundle, entity):
|
|
217
|
+
return sorted(
|
|
218
|
+
p["name"]
|
|
219
|
+
for e in bundle.conceptual_schema["entities"]
|
|
220
|
+
if e["name"] == entity
|
|
221
|
+
for p in e["properties"]
|
|
222
|
+
)
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def test_a_union_domain_gives_the_field_to_every_member():
|
|
226
|
+
bundle = parse_owl_with_rdflib(ANALYZER_OWL)
|
|
227
|
+
# The anonymous union class is not an entity of its own.
|
|
228
|
+
assert sorted(e["name"] for e in bundle.conceptual_schema["entities"]) == ["Chunk", "Document", "Note"]
|
|
229
|
+
assert _names(bundle, "Chunk") == ["content"]
|
|
230
|
+
assert _names(bundle, "Document") == ["content", "file_name", "tag"]
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def test_several_domains_each_get_the_field():
|
|
234
|
+
assert _names(parse_owl_with_rdflib(ANALYZER_OWL), "Note") == ["tag"]
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def test_attribute_name_becomes_the_physical_field():
|
|
238
|
+
bundle = parse_owl_with_rdflib(ANALYZER_OWL)
|
|
239
|
+
assert bundle.physical_mapping["entities"]["Document"]["properties"] == {
|
|
240
|
+
"file_name": {"field": "file-name"}
|
|
241
|
+
}
|
|
@@ -164,3 +164,91 @@ def test_export_label_style() -> None:
|
|
|
164
164
|
assert pm_p["style"] == "LABEL"
|
|
165
165
|
assert pm_p["typeField"] == "type"
|
|
166
166
|
assert pm_p["typeValue"] == "Person"
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
# ---------------------------------------------------------------------------
|
|
170
|
+
# Property names, shared fields and stored-field names (import fixes)
|
|
171
|
+
# ---------------------------------------------------------------------------
|
|
172
|
+
|
|
173
|
+
# The layout arangodb-schema-analyzer >= 0.15.1 writes: a field two classes
|
|
174
|
+
# share has an owl:unionOf domain; a renamed or sanitized field carries its
|
|
175
|
+
# stored name in phys:attributeName.
|
|
176
|
+
ANALYZER_OWL = """\
|
|
177
|
+
@prefix : <http://arangodb.com/schema/hybrid#> .
|
|
178
|
+
@prefix owl: <http://www.w3.org/2002/07/owl#> .
|
|
179
|
+
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
|
|
180
|
+
@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
|
|
181
|
+
@prefix phys: <http://arangodb.com/schema/physical#> .
|
|
182
|
+
|
|
183
|
+
:Chunk a owl:Class ;
|
|
184
|
+
rdfs:label "Chunk" .
|
|
185
|
+
:Chunk phys:mappingStyle "COLLECTION" .
|
|
186
|
+
:Chunk phys:collectionName "chunks" .
|
|
187
|
+
|
|
188
|
+
:Document a owl:Class ;
|
|
189
|
+
rdfs:label "Document" .
|
|
190
|
+
:Document phys:mappingStyle "COLLECTION" .
|
|
191
|
+
:Document phys:collectionName "documents" .
|
|
192
|
+
|
|
193
|
+
:content a owl:DatatypeProperty ;
|
|
194
|
+
rdfs:label "content" ;
|
|
195
|
+
rdfs:domain [ a owl:Class ; owl:unionOf ( :Chunk :Document ) ] .
|
|
196
|
+
|
|
197
|
+
:file_name a owl:DatatypeProperty ;
|
|
198
|
+
rdfs:label "file-name" ;
|
|
199
|
+
rdfs:domain :Document .
|
|
200
|
+
:file_name phys:attributeName "file-name" .
|
|
201
|
+
|
|
202
|
+
:page_count a owl:DatatypeProperty ;
|
|
203
|
+
rdfs:label "page_count" ;
|
|
204
|
+
rdfs:domain :Document ;
|
|
205
|
+
rdfs:range xsd:integer .
|
|
206
|
+
"""
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def _props(bundle: MappingBundle, entity: str) -> list[str]:
|
|
210
|
+
return [
|
|
211
|
+
p["name"]
|
|
212
|
+
for e in bundle.conceptual_schema["entities"]
|
|
213
|
+
if e["name"] == entity
|
|
214
|
+
for p in e["properties"]
|
|
215
|
+
]
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def test_snake_case_entity_properties_keep_their_names(simple_bundle: MappingBundle) -> None:
|
|
219
|
+
"""Regression: every name was cut at its last underscore (file_name -> name)."""
|
|
220
|
+
simple_bundle.conceptual_schema["entities"][0]["properties"].append(
|
|
221
|
+
{"name": "account_id", "type": "string"}
|
|
222
|
+
)
|
|
223
|
+
back = turtle_to_mapping(mapping_to_turtle(simple_bundle))
|
|
224
|
+
assert _props(back, "Person") == ["name", "age", "account_id"]
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def test_relationship_properties_lose_only_their_prefix(simple_bundle: MappingBundle) -> None:
|
|
228
|
+
simple_bundle.conceptual_schema["relationships"][0]["properties"] = [
|
|
229
|
+
{"name": "role_name", "type": "string"}
|
|
230
|
+
]
|
|
231
|
+
back = turtle_to_mapping(mapping_to_turtle(simple_bundle))
|
|
232
|
+
(rel,) = back.conceptual_schema["relationships"]
|
|
233
|
+
assert [p["name"] for p in rel["properties"]] == ["role_name"]
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def test_a_field_with_a_union_domain_belongs_to_every_member() -> None:
|
|
237
|
+
bundle = turtle_to_mapping(ANALYZER_OWL)
|
|
238
|
+
assert _props(bundle, "Chunk") == ["content"]
|
|
239
|
+
assert _props(bundle, "Document") == ["content", "file_name", "page_count"]
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def test_attribute_name_maps_the_property_to_its_stored_field() -> None:
|
|
243
|
+
bundle = turtle_to_mapping(ANALYZER_OWL)
|
|
244
|
+
assert bundle.physical_mapping["entities"]["Document"]["properties"] == {
|
|
245
|
+
"file_name": {"field": "file-name"}
|
|
246
|
+
}
|
|
247
|
+
assert "properties" not in bundle.physical_mapping["entities"]["Chunk"]
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def test_a_property_declared_once_per_domain_reaches_each(simple_bundle: MappingBundle) -> None:
|
|
251
|
+
"""mapping_to_turtle writes one block per entity for a shared name."""
|
|
252
|
+
simple_bundle.conceptual_schema["entities"][1]["properties"].append({"name": "name", "type": "string"})
|
|
253
|
+
back = turtle_to_mapping(mapping_to_turtle(simple_bundle))
|
|
254
|
+
assert "name" in _props(back, "Person") and "name" in _props(back, "Movie")
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
"""A workaround for a library we own names its upstream issue
|
|
2
|
+
(`.cursor/rules/fix-upstream-dont-fork.mdc`).
|
|
3
|
+
|
|
4
|
+
A gap in an owned library is fixed or filed in that library. When this repo
|
|
5
|
+
cannot wait for the release, the workaround is allowed only while it is marked
|
|
6
|
+
where it lives::
|
|
7
|
+
|
|
8
|
+
# upstream: <owner>/<repo>#<n> — remove when <condition>
|
|
9
|
+
|
|
10
|
+
so the pin bump that ships the fix can find it. This holds the marker's shape in
|
|
11
|
+
every scanned file: an issue a person can open, and a condition someone can
|
|
12
|
+
check. Whether that issue is still open needs the network, so it is not checked
|
|
13
|
+
here.
|
|
14
|
+
|
|
15
|
+
Shared source: arango-shared-memory ``templates/tests/test_upstream_markers.py``.
|
|
16
|
+
Copy changes back there so every repository keeps the same check.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import os
|
|
22
|
+
import re
|
|
23
|
+
from pathlib import Path
|
|
24
|
+
|
|
25
|
+
import pytest
|
|
26
|
+
|
|
27
|
+
ROOT = Path(__file__).resolve().parents[1]
|
|
28
|
+
RULE = ROOT / ".cursor" / "rules" / "fix-upstream-dont-fork.mdc"
|
|
29
|
+
|
|
30
|
+
#: Files whose comments start with ``#``.
|
|
31
|
+
SCANNED_SUFFIXES = frozenset({".py", ".sh", ".yml", ".yaml", ".toml", ".cfg"})
|
|
32
|
+
SCANNED_NAMES = frozenset({"Makefile", "Dockerfile"})
|
|
33
|
+
#: Directories that hold no code of ours. Hidden directories are skipped too,
|
|
34
|
+
#: except ``.github``, where CI steps can work around a library.
|
|
35
|
+
SKIPPED_DIRS = frozenset(
|
|
36
|
+
{
|
|
37
|
+
"__pycache__",
|
|
38
|
+
"build",
|
|
39
|
+
"dist",
|
|
40
|
+
"node_modules",
|
|
41
|
+
"site-packages",
|
|
42
|
+
"venv",
|
|
43
|
+
}
|
|
44
|
+
)
|
|
45
|
+
KEPT_HIDDEN_DIRS = frozenset({".github"})
|
|
46
|
+
|
|
47
|
+
#: Any comment that claims to be a marker, however it is written.
|
|
48
|
+
MARKER = re.compile(r"#\s*upstream\s*:(?P<rest>.*)$", re.IGNORECASE)
|
|
49
|
+
#: What a marker must say: an issue (owner/repo#n), then when to remove it.
|
|
50
|
+
WELL_FORMED = re.compile(
|
|
51
|
+
r"^\s*(?P<repo>[A-Za-z0-9][\w.-]*/[A-Za-z0-9][\w.-]*)#(?P<issue>[1-9]\d*)"
|
|
52
|
+
r"\s+(?:—|–|--?)\s+remove when\s+(?P<when>\S.*?)\s*$"
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def marker_problems(text: str, where: str) -> list[str]:
|
|
57
|
+
"""Each malformed ``upstream:`` marker in ``text``, as ``where:line: why``."""
|
|
58
|
+
problems = []
|
|
59
|
+
for number, line in enumerate(text.splitlines(), start=1):
|
|
60
|
+
found = MARKER.search(line)
|
|
61
|
+
if found is None:
|
|
62
|
+
continue
|
|
63
|
+
if WELL_FORMED.match(found.group("rest")) is None:
|
|
64
|
+
problems.append(
|
|
65
|
+
f"{where}:{number}: {line.strip()!r} — write it as "
|
|
66
|
+
"'# upstream: <owner>/<repo>#<n> — remove when <condition>'"
|
|
67
|
+
)
|
|
68
|
+
return problems
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _skipped_dir(name: str) -> bool:
|
|
72
|
+
if name in SKIPPED_DIRS or name.endswith(".egg-info"):
|
|
73
|
+
return True
|
|
74
|
+
return name.startswith(".") and name not in KEPT_HIDDEN_DIRS
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _skipped(relative: Path) -> bool:
|
|
78
|
+
return any(_skipped_dir(part) for part in relative.parts[:-1])
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _scanned_files() -> list[Path]:
|
|
82
|
+
files = []
|
|
83
|
+
# os.walk with pruning, so .git, virtualenvs and node_modules are never entered.
|
|
84
|
+
for current, dirs, names in os.walk(ROOT):
|
|
85
|
+
dirs[:] = sorted(name for name in dirs if not _skipped_dir(name))
|
|
86
|
+
for name in sorted(names):
|
|
87
|
+
path = Path(current) / name
|
|
88
|
+
if path.suffix in SCANNED_SUFFIXES or name in SCANNED_NAMES:
|
|
89
|
+
files.append(path)
|
|
90
|
+
# This file states malformed markers on purpose, to prove they are caught.
|
|
91
|
+
return [path for path in files if path.resolve() != Path(__file__).resolve()]
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
# -- the tree --------------------------------------------------------------------
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def test_every_upstream_marker_names_an_issue_and_when_to_remove_it() -> None:
|
|
98
|
+
problems = []
|
|
99
|
+
for path in _scanned_files():
|
|
100
|
+
text = path.read_text(encoding="utf-8", errors="replace")
|
|
101
|
+
problems.extend(marker_problems(text, str(path.relative_to(ROOT))))
|
|
102
|
+
assert not problems, "malformed upstream markers:\n " + "\n ".join(problems)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def test_the_scan_reaches_this_repositorys_code() -> None:
|
|
106
|
+
"""A scan that silently finds nothing would pass forever."""
|
|
107
|
+
scanned = {path.relative_to(ROOT).parts[0] for path in _scanned_files()}
|
|
108
|
+
assert "tests" in scanned
|
|
109
|
+
assert len(_scanned_files()) > 1
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
@pytest.mark.parametrize(
|
|
113
|
+
("relative", "skipped"),
|
|
114
|
+
[
|
|
115
|
+
("src/pkg/module.py", False),
|
|
116
|
+
(".github/workflows/ci.yml", False),
|
|
117
|
+
(".venv/lib/python3.11/site-packages/x.py", True),
|
|
118
|
+
("pkg.egg-info/PKG-INFO.py", True),
|
|
119
|
+
("frontend/node_modules/x/setup.py", True),
|
|
120
|
+
(".claude/hooks/x.py", True),
|
|
121
|
+
],
|
|
122
|
+
)
|
|
123
|
+
def test_the_scan_skips_only_what_is_not_ours(relative: str, skipped: bool) -> None:
|
|
124
|
+
assert _skipped(Path(relative)) is skipped
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
# -- the check itself ------------------------------------------------------------
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
@pytest.mark.parametrize(
|
|
131
|
+
"line",
|
|
132
|
+
[
|
|
133
|
+
"# upstream: ArthurKeen/arango-sparql-py#123 — remove when the engine takes a validator",
|
|
134
|
+
"x = build() # upstream: arango-solutions-org/r2g-arango#7 -- remove when 0.7 ships",
|
|
135
|
+
" # upstream: org/arango-query-core#42 – remove when LabelIndex accepts aliases",
|
|
136
|
+
" run: make gate # upstream: org/arango-cypher#9 - remove when the pin moves",
|
|
137
|
+
],
|
|
138
|
+
)
|
|
139
|
+
def test_a_well_formed_marker_passes(line: str) -> None:
|
|
140
|
+
assert marker_problems(line, "example") == []
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
@pytest.mark.parametrize(
|
|
144
|
+
("line", "missing"),
|
|
145
|
+
[
|
|
146
|
+
("# upstream: TODO file an issue", "no issue at all"),
|
|
147
|
+
("# upstream: arango-sparql-py#12 — remove when fixed", "no owner"),
|
|
148
|
+
("# upstream: org/arango-sparql-py — remove when fixed", "no issue number"),
|
|
149
|
+
("# upstream: org/arango-sparql-py#0 — remove when fixed", "issue zero"),
|
|
150
|
+
("# upstream: org/arango-sparql-py#12", "no removal condition"),
|
|
151
|
+
("# upstream: org/arango-sparql-py#12 — remove when ", "an empty condition"),
|
|
152
|
+
("# Upstream: org/arango-sparql-py#12 remove when fixed", "no separator"),
|
|
153
|
+
("#upstream:https://github.com/org/arango-sparql-py/issues/12", "a URL, not owner/repo#n"),
|
|
154
|
+
],
|
|
155
|
+
)
|
|
156
|
+
def test_a_malformed_marker_is_named_with_where_it_is(line: str, missing: str) -> None:
|
|
157
|
+
problems = marker_problems(f"first line\n{line}\n", "src/example.py")
|
|
158
|
+
assert len(problems) == 1, f"{missing}: {problems}"
|
|
159
|
+
assert problems[0].startswith("src/example.py:2: ")
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def test_a_line_that_only_mentions_upstream_is_not_a_marker() -> None:
|
|
163
|
+
text = "# fixed upstream in 0.6.1\n# the upstream repository leads\nupstream = 'x'\n"
|
|
164
|
+
assert marker_problems(text, "example") == []
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
# -- the rule says what this test holds ------------------------------------------
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def test_the_rule_names_this_test_and_the_marker_format() -> None:
|
|
171
|
+
rule = RULE.read_text(encoding="utf-8")
|
|
172
|
+
assert "tests/test_upstream_markers.py" in rule
|
|
173
|
+
assert "# upstream: <org>/" in rule and "remove when" in rule
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{arango_query_core-0.2.0 → arango_query_core-0.2.2}/tests/test_mapping_resolver_normalization.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|