cognite-neat 0.86.0__py3-none-any.whl → 0.87.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_version.py +1 -1
- cognite/neat/constants.py +11 -9
- cognite/neat/graph/extractors/_mock_graph_generator.py +7 -8
- cognite/neat/graph/loaders/__init__.py +5 -2
- cognite/neat/graph/loaders/_base.py +13 -5
- cognite/neat/graph/loaders/_rdf2asset.py +94 -20
- cognite/neat/graph/loaders/_rdf2dms.py +1 -1
- cognite/neat/graph/queries/_base.py +1 -1
- cognite/neat/graph/queries/_construct.py +2 -2
- cognite/neat/graph/queries/_shared.py +20 -6
- cognite/neat/graph/stores/_base.py +4 -3
- cognite/neat/legacy/graph/extractors/_dexpi.py +0 -5
- cognite/neat/legacy/graph/stores/_base.py +24 -8
- cognite/neat/legacy/graph/stores/_graphdb_store.py +3 -2
- cognite/neat/legacy/graph/stores/_memory_store.py +3 -3
- cognite/neat/legacy/graph/stores/_oxigraph_store.py +8 -4
- cognite/neat/legacy/graph/stores/_rdf_to_graph.py +5 -3
- cognite/neat/legacy/graph/transformations/query_generator/sparql.py +48 -15
- cognite/neat/legacy/rules/importers/_graph2rules.py +34 -7
- cognite/neat/legacy/rules/models/raw_rules.py +18 -6
- cognite/neat/legacy/rules/models/rules.py +32 -12
- cognite/neat/rules/_shared.py +6 -1
- cognite/neat/rules/analysis/__init__.py +4 -4
- cognite/neat/rules/analysis/_asset.py +128 -0
- cognite/neat/rules/analysis/_base.py +385 -6
- cognite/neat/rules/analysis/_information.py +155 -0
- cognite/neat/rules/exporters/_rules2ontology.py +4 -4
- cognite/neat/rules/importers/_dtdl2rules/dtdl_converter.py +2 -8
- cognite/neat/rules/importers/_inference2rules.py +2 -2
- cognite/neat/rules/models/_base.py +7 -7
- cognite/neat/rules/models/asset/_rules.py +4 -5
- cognite/neat/rules/models/dms/_converter.py +1 -2
- cognite/neat/rules/models/dms/_rules.py +3 -0
- cognite/neat/rules/models/domain.py +5 -2
- cognite/neat/rules/models/entities.py +2 -9
- cognite/neat/rules/models/information/_rules.py +10 -8
- cognite/neat/rules/models/information/_rules_input.py +1 -2
- cognite/neat/rules/models/information/_validation.py +1 -1
- cognite/neat/workflows/steps/lib/current/graph_store.py +28 -8
- cognite/neat/workflows/steps/lib/legacy/graph_extractor.py +129 -27
- cognite/neat/workflows/steps/lib/legacy/graph_store.py +4 -4
- {cognite_neat-0.86.0.dist-info → cognite_neat-0.87.0.dist-info}/METADATA +1 -1
- {cognite_neat-0.86.0.dist-info → cognite_neat-0.87.0.dist-info}/RECORD +46 -45
- cognite/neat/rules/analysis/_information_rules.py +0 -476
- {cognite_neat-0.86.0.dist-info → cognite_neat-0.87.0.dist-info}/LICENSE +0 -0
- {cognite_neat-0.86.0.dist-info → cognite_neat-0.87.0.dist-info}/WHEEL +0 -0
- {cognite_neat-0.86.0.dist-info → cognite_neat-0.87.0.dist-info}/entry_points.txt +0 -0
|
@@ -2,7 +2,7 @@ from pathlib import Path
|
|
|
2
2
|
|
|
3
3
|
from rdflib import Graph, Namespace
|
|
4
4
|
|
|
5
|
-
from cognite.neat.constants import
|
|
5
|
+
from cognite.neat.constants import get_default_prefixes
|
|
6
6
|
|
|
7
7
|
|
|
8
8
|
def rdf_file_to_graph(
|
|
@@ -10,7 +10,7 @@ def rdf_file_to_graph(
|
|
|
10
10
|
filepath: Path,
|
|
11
11
|
base_prefix: str | None = None,
|
|
12
12
|
base_namespace: Namespace | None = None,
|
|
13
|
-
prefixes: dict[str, Namespace] =
|
|
13
|
+
prefixes: dict[str, Namespace] | None = None,
|
|
14
14
|
) -> Graph:
|
|
15
15
|
"""Created rdflib Graph instance loaded with RDF triples from file
|
|
16
16
|
|
|
@@ -18,13 +18,15 @@ def rdf_file_to_graph(
|
|
|
18
18
|
filepath: Path to the RDF file
|
|
19
19
|
base_prefix: base prefix for URIs. Defaults to None.
|
|
20
20
|
base_namespace: base namespace for URIs . Defaults to None.
|
|
21
|
-
prefixes: Dictionary of prefixes to bind to graph. Defaults to
|
|
21
|
+
prefixes: Dictionary of prefixes to bind to graph. Defaults to internal set of prefixes.
|
|
22
22
|
graph: Graph instance to load RDF triples into. Defaults to None.
|
|
23
23
|
|
|
24
24
|
Returns:
|
|
25
25
|
Graph instance loaded with RDF triples from file
|
|
26
26
|
"""
|
|
27
27
|
|
|
28
|
+
prefixes = prefixes if prefixes else get_default_prefixes()
|
|
29
|
+
|
|
28
30
|
if filepath.is_file():
|
|
29
31
|
graph.parse(filepath, publicID=base_namespace)
|
|
30
32
|
else:
|
|
@@ -6,7 +6,7 @@ from typing import cast
|
|
|
6
6
|
from rdflib import Graph, Namespace
|
|
7
7
|
from rdflib.term import URIRef
|
|
8
8
|
|
|
9
|
-
from cognite.neat.constants import
|
|
9
|
+
from cognite.neat.constants import get_default_prefixes
|
|
10
10
|
from cognite.neat.legacy.rules.analysis import get_classes_with_properties
|
|
11
11
|
from cognite.neat.legacy.rules.models._base import Triple
|
|
12
12
|
from cognite.neat.legacy.rules.models.rdfpath import (
|
|
@@ -24,7 +24,7 @@ from cognite.neat.legacy.rules.models.rules import Rules
|
|
|
24
24
|
from cognite.neat.utils.utils import remove_namespace_from_uri
|
|
25
25
|
|
|
26
26
|
|
|
27
|
-
def _generate_prefix_header(prefixes: dict[str, Namespace] =
|
|
27
|
+
def _generate_prefix_header(prefixes: dict[str, Namespace] | None = None) -> str:
|
|
28
28
|
"""Generate prefix header which is added to SPARQL query and allows for shorten query statements
|
|
29
29
|
|
|
30
30
|
Parameters
|
|
@@ -37,11 +37,15 @@ def _generate_prefix_header(prefixes: dict[str, Namespace] = PREFIXES) -> str:
|
|
|
37
37
|
str
|
|
38
38
|
Prefix header
|
|
39
39
|
"""
|
|
40
|
+
prefixes = prefixes if prefixes else get_default_prefixes()
|
|
40
41
|
return "".join(f"PREFIX {key}:<{value}>\n" for key, value in prefixes.items())
|
|
41
42
|
|
|
42
43
|
|
|
43
44
|
def _get_predicate_id(
|
|
44
|
-
graph: Graph,
|
|
45
|
+
graph: Graph,
|
|
46
|
+
subject_type_id: str,
|
|
47
|
+
object_type_id: str,
|
|
48
|
+
prefixes: dict[str, Namespace] | None = None,
|
|
45
49
|
) -> URIRef:
|
|
46
50
|
"""Returns predicate (aka property) URI (i.e., ID) that connects subject and object
|
|
47
51
|
|
|
@@ -61,6 +65,7 @@ def _get_predicate_id(
|
|
|
61
65
|
URIRef
|
|
62
66
|
ID of predicate (aka property) connecting subject and object
|
|
63
67
|
"""
|
|
68
|
+
prefixes = prefixes if prefixes else get_default_prefixes()
|
|
64
69
|
query = """
|
|
65
70
|
|
|
66
71
|
SELECT ?predicateTypeID
|
|
@@ -122,11 +127,21 @@ def _get_hop_triples(graph, path: Hop, prefixes) -> list[Triple]:
|
|
|
122
127
|
if previous_step.property:
|
|
123
128
|
triples.extend(
|
|
124
129
|
[
|
|
125
|
-
Triple(subject=f"?{previous_step.class_.suffix}ID", predicate="a", object=previous_step.class_.id),
|
|
126
130
|
Triple(
|
|
127
|
-
subject=f"?{previous_step.class_.suffix}ID",
|
|
131
|
+
subject=f"?{previous_step.class_.suffix}ID",
|
|
132
|
+
predicate="a",
|
|
133
|
+
object=previous_step.class_.id,
|
|
134
|
+
),
|
|
135
|
+
Triple(
|
|
136
|
+
subject=f"?{previous_step.class_.suffix}ID",
|
|
137
|
+
predicate=previous_step.property.id,
|
|
138
|
+
object="?object",
|
|
139
|
+
),
|
|
140
|
+
Triple(
|
|
141
|
+
subject="?predicate",
|
|
142
|
+
predicate="a",
|
|
143
|
+
object=previous_step.property.id,
|
|
128
144
|
),
|
|
129
|
-
Triple(subject="?predicate", predicate="a", object=previous_step.property.id),
|
|
130
145
|
]
|
|
131
146
|
)
|
|
132
147
|
else:
|
|
@@ -139,7 +154,7 @@ def _get_hop_triples(graph, path: Hop, prefixes) -> list[Triple]:
|
|
|
139
154
|
return triples
|
|
140
155
|
|
|
141
156
|
|
|
142
|
-
def _get_path_triples(graph: Graph, traversal: Traversal, prefixes: dict[str, Namespace] =
|
|
157
|
+
def _get_path_triples(graph: Graph, traversal: Traversal, prefixes: dict[str, Namespace] | None = None) -> list[Triple]:
|
|
143
158
|
"""Creates triples subject-predicate-object from declarative graph traversal path
|
|
144
159
|
|
|
145
160
|
Parameters
|
|
@@ -227,7 +242,9 @@ def _generate_all_references_query_statement(
|
|
|
227
242
|
|
|
228
243
|
|
|
229
244
|
def _generate_single_property_query_statement(
|
|
230
|
-
subject: str,
|
|
245
|
+
subject: str,
|
|
246
|
+
predicate: str,
|
|
247
|
+
query_template: str = SINGLE_PROPERTY_SPARQL_QUERY_TEMPLATE,
|
|
231
248
|
) -> str:
|
|
232
249
|
query_insertions = "\n".join([f"\t\t?subject a {subject} .", f"\t\t?subject {predicate} ?object ."])
|
|
233
250
|
|
|
@@ -257,7 +274,7 @@ def _generate_hop_query_statement(triples: list[Triple], query_template: str = B
|
|
|
257
274
|
def build_sparql_query(
|
|
258
275
|
graph: Graph,
|
|
259
276
|
traversal_path: str | Traversal,
|
|
260
|
-
prefixes: dict[str, Namespace] =
|
|
277
|
+
prefixes: dict[str, Namespace] | None = None,
|
|
261
278
|
insert_prefixes: bool = False,
|
|
262
279
|
) -> str:
|
|
263
280
|
"""Builds SPARQL query based on declarative traversal path
|
|
@@ -276,7 +293,7 @@ def build_sparql_query(
|
|
|
276
293
|
str
|
|
277
294
|
SPARQL query
|
|
278
295
|
"""
|
|
279
|
-
|
|
296
|
+
prefixes = prefixes if prefixes else get_default_prefixes()
|
|
280
297
|
traversal = parse_traversal(traversal_path) if isinstance(traversal_path, str) else traversal_path
|
|
281
298
|
triples = _get_path_triples(graph, traversal, prefixes)
|
|
282
299
|
|
|
@@ -297,7 +314,10 @@ def build_sparql_query(
|
|
|
297
314
|
for prefix, URI in prefixes.items():
|
|
298
315
|
query = query.replace(URI, f"{prefix}:")
|
|
299
316
|
|
|
300
|
-
return query.replace(
|
|
317
|
+
return query.replace(
|
|
318
|
+
"insertPrefixes\n\n",
|
|
319
|
+
_generate_prefix_header(prefixes) if insert_prefixes else "",
|
|
320
|
+
)
|
|
301
321
|
|
|
302
322
|
|
|
303
323
|
def compress_uri(uri: URIRef, prefixes: dict) -> str:
|
|
@@ -420,7 +440,8 @@ def _add_filter(class_instances, query_template):
|
|
|
420
440
|
if class_instances:
|
|
421
441
|
class_instances_formatted = [f"<{instance}>" for instance in class_instances]
|
|
422
442
|
query_template = query_template.replace(
|
|
423
|
-
"insert_filter",
|
|
443
|
+
"insert_filter",
|
|
444
|
+
f"\n\nFILTER (?subject IN ({', '.join(class_instances_formatted)}))",
|
|
424
445
|
)
|
|
425
446
|
else:
|
|
426
447
|
query_template = query_template.replace("insert_filter", "")
|
|
@@ -439,7 +460,10 @@ def _triples2sparql_statement(triples: list[Triple]):
|
|
|
439
460
|
|
|
440
461
|
|
|
441
462
|
def _to_construct_triples(
|
|
442
|
-
graph: Graph,
|
|
463
|
+
graph: Graph,
|
|
464
|
+
class_: str,
|
|
465
|
+
transformation_rules: Rules,
|
|
466
|
+
properties_optional: bool = True,
|
|
443
467
|
) -> tuple[list[Triple], list[Triple]]:
|
|
444
468
|
"""Converts class definition to CONSTRUCT triples which are used to generate CONSTRUCT query
|
|
445
469
|
|
|
@@ -485,7 +509,10 @@ def _to_construct_triples(
|
|
|
485
509
|
# by binding them to certain property
|
|
486
510
|
if isinstance(traversal, AllReferences):
|
|
487
511
|
graph_pattern_triple = Triple(
|
|
488
|
-
subject="BIND(?subject",
|
|
512
|
+
subject="BIND(?subject",
|
|
513
|
+
predicate="AS",
|
|
514
|
+
object=f"{graph_template_triple.object})",
|
|
515
|
+
optional=False,
|
|
489
516
|
)
|
|
490
517
|
|
|
491
518
|
elif isinstance(traversal, SingleProperty):
|
|
@@ -510,7 +537,13 @@ def _to_construct_triples(
|
|
|
510
537
|
|
|
511
538
|
# add first triple for graph pattern stating type of object
|
|
512
539
|
patterns.insert(
|
|
513
|
-
0,
|
|
540
|
+
0,
|
|
541
|
+
Triple(
|
|
542
|
+
subject="?subject",
|
|
543
|
+
predicate="a",
|
|
544
|
+
object=_most_occurring_element(class_ids),
|
|
545
|
+
optional=False,
|
|
546
|
+
),
|
|
514
547
|
)
|
|
515
548
|
|
|
516
549
|
return templates, patterns
|
|
@@ -10,11 +10,15 @@ from typing import cast
|
|
|
10
10
|
import pandas as pd
|
|
11
11
|
from rdflib import Graph, Literal, Namespace, URIRef
|
|
12
12
|
|
|
13
|
-
from cognite.neat.constants import
|
|
13
|
+
from cognite.neat.constants import get_default_prefixes
|
|
14
14
|
from cognite.neat.legacy.rules import exceptions
|
|
15
15
|
from cognite.neat.legacy.rules.exporters._rules2rules import to_dms_name
|
|
16
16
|
from cognite.neat.legacy.rules.models.tables import Tables
|
|
17
|
-
from cognite.neat.utils.utils import
|
|
17
|
+
from cognite.neat.utils.utils import (
|
|
18
|
+
get_namespace,
|
|
19
|
+
remove_namespace_from_uri,
|
|
20
|
+
uri_to_short_form,
|
|
21
|
+
)
|
|
18
22
|
|
|
19
23
|
from ._base import BaseImporter
|
|
20
24
|
|
|
@@ -74,11 +78,26 @@ def _create_default_properties_parsing_config() -> dict[str, tuple[str, ...]]:
|
|
|
74
78
|
|
|
75
79
|
def _create_default_classes_parsing_config() -> dict[str, tuple[str, ...]]:
|
|
76
80
|
# TODO: these are to be read from Class pydantic model
|
|
77
|
-
return {
|
|
81
|
+
return {
|
|
82
|
+
"header": (
|
|
83
|
+
"Class",
|
|
84
|
+
"Description",
|
|
85
|
+
"Parent Class",
|
|
86
|
+
"Source",
|
|
87
|
+
"Source Entity Name",
|
|
88
|
+
"Match Type",
|
|
89
|
+
"Comment",
|
|
90
|
+
)
|
|
91
|
+
}
|
|
78
92
|
|
|
79
93
|
|
|
80
94
|
def _parse_prefixes_df(prefixes: dict[str, Namespace]) -> pd.DataFrame:
|
|
81
|
-
return pd.DataFrame.from_dict(
|
|
95
|
+
return pd.DataFrame.from_dict(
|
|
96
|
+
{
|
|
97
|
+
"Prefix": list(prefixes.keys()),
|
|
98
|
+
"URI": [str(uri) for uri in prefixes.values()],
|
|
99
|
+
}
|
|
100
|
+
)
|
|
82
101
|
|
|
83
102
|
|
|
84
103
|
def _parse_metadata_df() -> pd.DataFrame:
|
|
@@ -169,7 +188,7 @@ def _graph_to_data_model_dict(graph: Graph, max_number_of_instance: int = -1) ->
|
|
|
169
188
|
"""
|
|
170
189
|
data_model: dict[str, dict] = {}
|
|
171
190
|
|
|
172
|
-
prefixes: dict[str, Namespace] =
|
|
191
|
+
prefixes: dict[str, Namespace] = get_default_prefixes()
|
|
173
192
|
|
|
174
193
|
for class_ in _get_class_ids(graph):
|
|
175
194
|
_add_uri_namespace_to_prefixes(class_, prefixes)
|
|
@@ -183,10 +202,18 @@ def _graph_to_data_model_dict(graph: Graph, max_number_of_instance: int = -1) ->
|
|
|
183
202
|
)
|
|
184
203
|
class_name = f"{class_name}_{len(data_model)+1}"
|
|
185
204
|
|
|
186
|
-
data_model[class_name] = {
|
|
205
|
+
data_model[class_name] = {
|
|
206
|
+
"properties": {},
|
|
207
|
+
"uri": uri_to_short_form(class_, prefixes),
|
|
208
|
+
}
|
|
187
209
|
|
|
188
210
|
for instance in _get_class_instance_ids(graph, class_, max_number_of_instance):
|
|
189
|
-
for
|
|
211
|
+
for (
|
|
212
|
+
property_,
|
|
213
|
+
occurrence,
|
|
214
|
+
data_type,
|
|
215
|
+
object_type,
|
|
216
|
+
) in _define_instance_properties(graph, instance):
|
|
190
217
|
property_name = remove_namespace_from_uri(property_)
|
|
191
218
|
_add_uri_namespace_to_prefixes(property_, prefixes)
|
|
192
219
|
|
|
@@ -9,11 +9,17 @@ from pydantic import field_validator
|
|
|
9
9
|
from pydantic_core import ErrorDetails, ValidationError
|
|
10
10
|
from rdflib import Namespace
|
|
11
11
|
|
|
12
|
-
from cognite.neat.constants import
|
|
12
|
+
from cognite.neat.constants import get_default_prefixes
|
|
13
13
|
from cognite.neat.exceptions import wrangle_warnings
|
|
14
14
|
|
|
15
15
|
# rules model and model components:
|
|
16
|
-
from cognite.neat.legacy.rules.models.rules import
|
|
16
|
+
from cognite.neat.legacy.rules.models.rules import (
|
|
17
|
+
Class,
|
|
18
|
+
Metadata,
|
|
19
|
+
Property,
|
|
20
|
+
RuleModel,
|
|
21
|
+
Rules,
|
|
22
|
+
)
|
|
17
23
|
from cognite.neat.legacy.rules.models.tables import Tables
|
|
18
24
|
|
|
19
25
|
# importers:
|
|
@@ -36,7 +42,7 @@ class RawRules(RuleModel):
|
|
|
36
42
|
classes: Classes defined in the data model
|
|
37
43
|
properties: Class properties defined in the data model with accompanying transformation rules
|
|
38
44
|
to transform data from source to target representation
|
|
39
|
-
prefixes: Prefixes used in the data model. Defaults to
|
|
45
|
+
prefixes: Prefixes used in the data model. Defaults to internal set of prefixes
|
|
40
46
|
instances: Instances defined in the data model. Defaults to None
|
|
41
47
|
"""
|
|
42
48
|
|
|
@@ -242,7 +248,9 @@ def _raw_tables_to_rules_dict(raw_tables: RawRules, validators_to_skip: set | No
|
|
|
242
248
|
"metadata": _metadata_table2dict(raw_tables.Metadata),
|
|
243
249
|
"classes": _classes_table2dict(raw_tables.Classes),
|
|
244
250
|
"properties": _properties_table2dict(raw_tables.Properties),
|
|
245
|
-
"prefixes":
|
|
251
|
+
"prefixes": (
|
|
252
|
+
get_default_prefixes() if raw_tables.Prefixes.empty else _prefixes_table2dict(raw_tables.Prefixes)
|
|
253
|
+
),
|
|
246
254
|
}
|
|
247
255
|
|
|
248
256
|
rules_dict["instances"] = (
|
|
@@ -272,11 +280,15 @@ def _metadata_table2dict(meta_df: pd.DataFrame) -> dict[str, Any]:
|
|
|
272
280
|
return metadata_dict
|
|
273
281
|
|
|
274
282
|
|
|
275
|
-
def _classes_table2dict(
|
|
283
|
+
def _classes_table2dict(
|
|
284
|
+
classes_df: pd.DataFrame,
|
|
285
|
+
) -> dict[Any | None, dict[Hashable, Any]]:
|
|
276
286
|
return {class_.get("Class"): class_ for class_ in classes_df.to_dict(orient="records")}
|
|
277
287
|
|
|
278
288
|
|
|
279
|
-
def _properties_table2dict(
|
|
289
|
+
def _properties_table2dict(
|
|
290
|
+
properties_df: pd.DataFrame,
|
|
291
|
+
) -> dict[str, dict[Hashable, Any]]:
|
|
280
292
|
return {f"row {i+3}": property_ for i, property_ in enumerate(properties_df.to_dict(orient="records"))}
|
|
281
293
|
|
|
282
294
|
|
|
@@ -29,7 +29,7 @@ from pydantic import (
|
|
|
29
29
|
from pydantic.fields import FieldInfo
|
|
30
30
|
from rdflib import XSD, Literal, Namespace, URIRef
|
|
31
31
|
|
|
32
|
-
from cognite.neat.constants import
|
|
32
|
+
from cognite.neat.constants import get_default_prefixes
|
|
33
33
|
from cognite.neat.legacy.rules import exceptions
|
|
34
34
|
from cognite.neat.legacy.rules.models._base import (
|
|
35
35
|
ENTITY_ID_REGEX_COMPILED,
|
|
@@ -49,7 +49,10 @@ from cognite.neat.legacy.rules.models.rdfpath import (
|
|
|
49
49
|
Traversal,
|
|
50
50
|
parse_rule,
|
|
51
51
|
)
|
|
52
|
-
from cognite.neat.legacy.rules.models.value_types import
|
|
52
|
+
from cognite.neat.legacy.rules.models.value_types import (
|
|
53
|
+
XSD_VALUE_TYPE_MAPPINGS,
|
|
54
|
+
ValueType,
|
|
55
|
+
)
|
|
53
56
|
|
|
54
57
|
if sys.version_info >= (3, 11):
|
|
55
58
|
from typing import Self
|
|
@@ -82,7 +85,10 @@ def replace_nan_floats_with_default(values: dict, model_fields: dict[str, FieldI
|
|
|
82
85
|
output[field_name] = model_fields[field_name].default
|
|
83
86
|
else:
|
|
84
87
|
# field_name may be an alias
|
|
85
|
-
source_name = next(
|
|
88
|
+
source_name = next(
|
|
89
|
+
(name for name, field in model_fields.items() if field.alias == field_name),
|
|
90
|
+
None,
|
|
91
|
+
)
|
|
86
92
|
if source_name:
|
|
87
93
|
output[field_name] = model_fields[source_name].default
|
|
88
94
|
else:
|
|
@@ -335,7 +341,9 @@ class Metadata(RuleModel):
|
|
|
335
341
|
if value.endswith("#") or value.endswith("/"):
|
|
336
342
|
return value
|
|
337
343
|
warnings.warn(
|
|
338
|
-
exceptions.NamespaceEndingFixed(value).message,
|
|
344
|
+
exceptions.NamespaceEndingFixed(value).message,
|
|
345
|
+
category=exceptions.NamespaceEndingFixed,
|
|
346
|
+
stacklevel=2,
|
|
339
347
|
)
|
|
340
348
|
return Namespace(f"{value}#")
|
|
341
349
|
|
|
@@ -444,10 +452,14 @@ class Resource(RuleModel):
|
|
|
444
452
|
default=None,
|
|
445
453
|
)
|
|
446
454
|
source_entity_name: str | None = Field(
|
|
447
|
-
alias="Source Entity Name",
|
|
455
|
+
alias="Source Entity Name",
|
|
456
|
+
description="Closest entity in source, e.g. Substation",
|
|
457
|
+
default=None,
|
|
448
458
|
)
|
|
449
459
|
match_type: str | None = Field(
|
|
450
|
-
alias="Match Type",
|
|
460
|
+
alias="Match Type",
|
|
461
|
+
description="Type of match between source entity and one being defined",
|
|
462
|
+
default=None,
|
|
451
463
|
)
|
|
452
464
|
comment: str | None = Field(alias="Comment", description="Comment about mapping", default=None)
|
|
453
465
|
|
|
@@ -656,7 +668,9 @@ class Property(Resource):
|
|
|
656
668
|
# Specialization of cdf_resource_type to allow definition of both
|
|
657
669
|
# Asset and Relationship at the same time
|
|
658
670
|
cdf_resource_type: list[str] = Field(
|
|
659
|
-
alias="Resource Type",
|
|
671
|
+
alias="Resource Type",
|
|
672
|
+
default_factory=list,
|
|
673
|
+
description="This is typically 'Asset' or 'Relationship'",
|
|
660
674
|
)
|
|
661
675
|
|
|
662
676
|
# Transformation rule (domain to solution)
|
|
@@ -701,7 +715,11 @@ class Property(Resource):
|
|
|
701
715
|
return ValueType.from_string(entity_string=value, type_=EntityTypes.object_value_type, mapping=None)
|
|
702
716
|
else:
|
|
703
717
|
return ValueType(
|
|
704
|
-
prefix="undefined",
|
|
718
|
+
prefix="undefined",
|
|
719
|
+
suffix=value,
|
|
720
|
+
name=value,
|
|
721
|
+
type_=EntityTypes.object_value_type,
|
|
722
|
+
mapping=None,
|
|
705
723
|
)
|
|
706
724
|
# return ValueType(
|
|
707
725
|
|
|
@@ -820,7 +838,9 @@ class Property(Resource):
|
|
|
820
838
|
def set_relationship_label(self):
|
|
821
839
|
if self.label is None:
|
|
822
840
|
warnings.warn(
|
|
823
|
-
exceptions.MissingLabel(self.property_id).message,
|
|
841
|
+
exceptions.MissingLabel(self.property_id).message,
|
|
842
|
+
category=exceptions.MissingLabel,
|
|
843
|
+
stacklevel=2,
|
|
824
844
|
)
|
|
825
845
|
self.label = self.property_id
|
|
826
846
|
return self
|
|
@@ -899,7 +919,7 @@ class Prefixes(RuleModel):
|
|
|
899
919
|
prefixes: Dict of prefixes
|
|
900
920
|
"""
|
|
901
921
|
|
|
902
|
-
prefixes: dict[str, Namespace] =
|
|
922
|
+
prefixes: dict[str, Namespace] = get_default_prefixes()
|
|
903
923
|
|
|
904
924
|
|
|
905
925
|
class Instance(RuleModel):
|
|
@@ -1019,7 +1039,7 @@ class Rules(RuleModel):
|
|
|
1019
1039
|
classes: Classes defined in the data model
|
|
1020
1040
|
properties: Class properties defined in the data model with accompanying transformation rules
|
|
1021
1041
|
to transform data from source to target representation
|
|
1022
|
-
prefixes: Prefixes used in the data model. Defaults to
|
|
1042
|
+
prefixes: Prefixes used in the data model. Defaults to internal prefixes
|
|
1023
1043
|
instances: Instances defined in the data model. Defaults to None
|
|
1024
1044
|
validators_to_skip: List of validators to skip. Defaults to []
|
|
1025
1045
|
|
|
@@ -1035,7 +1055,7 @@ class Rules(RuleModel):
|
|
|
1035
1055
|
metadata: Metadata
|
|
1036
1056
|
classes: Classes
|
|
1037
1057
|
properties: Properties
|
|
1038
|
-
prefixes: dict[str, Namespace] =
|
|
1058
|
+
prefixes: dict[str, Namespace] = get_default_prefixes()
|
|
1039
1059
|
instances: list[Instance] = Field(default_factory=list)
|
|
1040
1060
|
|
|
1041
1061
|
@property
|
cognite/neat/rules/_shared.py
CHANGED
|
@@ -1,5 +1,10 @@
|
|
|
1
1
|
from typing import TypeAlias
|
|
2
2
|
|
|
3
|
-
from cognite.neat.rules.models import
|
|
3
|
+
from cognite.neat.rules.models import (
|
|
4
|
+
AssetRules,
|
|
5
|
+
DMSRules,
|
|
6
|
+
DomainRules,
|
|
7
|
+
InformationRules,
|
|
8
|
+
)
|
|
4
9
|
|
|
5
10
|
Rules: TypeAlias = DomainRules | InformationRules | DMSRules | AssetRules
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
from .
|
|
2
|
-
|
|
3
|
-
|
|
1
|
+
from ._asset import AssetAnalysis
|
|
2
|
+
from ._information import (
|
|
3
|
+
InformationAnalysis,
|
|
4
4
|
)
|
|
5
5
|
|
|
6
|
-
__all__ = ["
|
|
6
|
+
__all__ = ["InformationAnalysis", "AssetAnalysis"]
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import warnings
|
|
2
|
+
from typing import cast
|
|
3
|
+
|
|
4
|
+
from cognite.neat.rules.models import AssetRules
|
|
5
|
+
from cognite.neat.rules.models._rdfpath import RDFPath
|
|
6
|
+
from cognite.neat.rules.models.asset import AssetClass, AssetProperty
|
|
7
|
+
from cognite.neat.rules.models.entities import (
|
|
8
|
+
AssetEntity,
|
|
9
|
+
ClassEntity,
|
|
10
|
+
EntityTypes,
|
|
11
|
+
ReferenceEntity,
|
|
12
|
+
RelationshipEntity,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
from ._base import BaseAnalysis
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class AssetAnalysis(BaseAnalysis[AssetRules, AssetClass, AssetProperty, ClassEntity, str]):
|
|
19
|
+
"""Assumes analysis over only the complete schema"""
|
|
20
|
+
|
|
21
|
+
def _get_reference(self, class_or_property: AssetClass | AssetProperty) -> ReferenceEntity | None:
|
|
22
|
+
return class_or_property.reference if isinstance(class_or_property.reference, ReferenceEntity) else None
|
|
23
|
+
|
|
24
|
+
def _get_cls_entity(self, class_: AssetClass | AssetProperty) -> ClassEntity:
|
|
25
|
+
return class_.class_
|
|
26
|
+
|
|
27
|
+
def _get_prop_entity(self, property_: AssetProperty) -> str:
|
|
28
|
+
return property_.property_
|
|
29
|
+
|
|
30
|
+
def _get_cls_parents(self, class_: AssetClass) -> list[ClassEntity] | None:
|
|
31
|
+
return list(class_.parent or []) or None
|
|
32
|
+
|
|
33
|
+
def _get_reference_rules(self) -> AssetRules | None:
|
|
34
|
+
return self.rules.reference
|
|
35
|
+
|
|
36
|
+
@classmethod
|
|
37
|
+
def _set_cls_entity(cls, property_: AssetProperty, class_: ClassEntity) -> None:
|
|
38
|
+
property_.class_ = class_
|
|
39
|
+
|
|
40
|
+
def _get_object(self, property_: AssetProperty) -> ClassEntity | None:
|
|
41
|
+
return property_.value_type if isinstance(property_.value_type, ClassEntity) else None
|
|
42
|
+
|
|
43
|
+
def _get_max_occurrence(self, property_: AssetProperty) -> int | float | None:
|
|
44
|
+
return property_.max_count
|
|
45
|
+
|
|
46
|
+
def _get_classes(self) -> list[AssetClass]:
|
|
47
|
+
return list(self.rules.classes)
|
|
48
|
+
|
|
49
|
+
def _get_properties(self) -> list[AssetProperty]:
|
|
50
|
+
return list(self.rules.properties)
|
|
51
|
+
|
|
52
|
+
def subset_rules(self, desired_classes: set[ClassEntity]) -> AssetRules:
|
|
53
|
+
raise NotImplementedError("Method not implemented")
|
|
54
|
+
|
|
55
|
+
def class_property_pairs(
|
|
56
|
+
self,
|
|
57
|
+
only_rdfpath: bool = False,
|
|
58
|
+
consider_inheritance: bool = False,
|
|
59
|
+
implementation_type: EntityTypes = EntityTypes.asset,
|
|
60
|
+
) -> dict[ClassEntity, dict[str, AssetProperty]]:
|
|
61
|
+
class_property_pairs = {}
|
|
62
|
+
|
|
63
|
+
T_implementation = AssetEntity if implementation_type == EntityTypes.asset else RelationshipEntity
|
|
64
|
+
|
|
65
|
+
for class_, properties in self.classes_with_properties(consider_inheritance).items():
|
|
66
|
+
processed_properties = {}
|
|
67
|
+
for property_ in properties:
|
|
68
|
+
if property_.property_ in processed_properties:
|
|
69
|
+
# TODO: use appropriate Warning class from _exceptions.py
|
|
70
|
+
# if missing make one !
|
|
71
|
+
warnings.warn(
|
|
72
|
+
f"Property {property_.property_} for {class_} has been defined more than once!"
|
|
73
|
+
" Only the first definition will be considered, skipping the rest..",
|
|
74
|
+
stacklevel=2,
|
|
75
|
+
)
|
|
76
|
+
continue
|
|
77
|
+
|
|
78
|
+
if (
|
|
79
|
+
property_.implementation
|
|
80
|
+
and any(isinstance(implementation, T_implementation) for implementation in property_.implementation)
|
|
81
|
+
and (not only_rdfpath or (only_rdfpath and isinstance(property_.transformation, RDFPath)))
|
|
82
|
+
):
|
|
83
|
+
implementation = [
|
|
84
|
+
implementation
|
|
85
|
+
for implementation in property_.implementation
|
|
86
|
+
if isinstance(implementation, T_implementation)
|
|
87
|
+
]
|
|
88
|
+
|
|
89
|
+
processed_properties[property_.property_] = property_.model_copy(
|
|
90
|
+
deep=True, update={"implementation": implementation}
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
if processed_properties:
|
|
94
|
+
class_property_pairs[class_] = processed_properties
|
|
95
|
+
|
|
96
|
+
return class_property_pairs
|
|
97
|
+
|
|
98
|
+
def asset_definition(
|
|
99
|
+
self, only_rdfpath: bool = False, consider_inheritance: bool = False
|
|
100
|
+
) -> dict[ClassEntity, dict[str, AssetProperty]]:
|
|
101
|
+
return self.class_property_pairs(
|
|
102
|
+
consider_inheritance=consider_inheritance,
|
|
103
|
+
only_rdfpath=only_rdfpath,
|
|
104
|
+
implementation_type=EntityTypes.asset,
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
def relationship_definition(
|
|
108
|
+
self, only_rdfpath: bool = False, consider_inheritance: bool = False
|
|
109
|
+
) -> dict[ClassEntity, dict[str, AssetProperty]]:
|
|
110
|
+
return self.class_property_pairs(
|
|
111
|
+
consider_inheritance=consider_inheritance,
|
|
112
|
+
only_rdfpath=only_rdfpath,
|
|
113
|
+
implementation_type=EntityTypes.relationship,
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
def define_property_renaming_config(self, class_: ClassEntity) -> dict[str, str]:
|
|
117
|
+
property_renaming_configuration = {}
|
|
118
|
+
|
|
119
|
+
if asset_definition := self.asset_definition().get(class_, None):
|
|
120
|
+
for property_, transformation in asset_definition.items():
|
|
121
|
+
asset_property = cast(list[AssetEntity], transformation.implementation)[0].property_
|
|
122
|
+
|
|
123
|
+
if asset_property != "metadata":
|
|
124
|
+
property_renaming_configuration[property_] = str(asset_property)
|
|
125
|
+
else:
|
|
126
|
+
property_renaming_configuration[property_] = f"{asset_property}.{property_}"
|
|
127
|
+
|
|
128
|
+
return property_renaming_configuration
|