cognite-neat 0.85.12__py3-none-any.whl → 0.87.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_version.py +1 -1
- cognite/neat/app/api/routers/core.py +4 -4
- cognite/neat/constants.py +11 -9
- cognite/neat/graph/extractors/_mock_graph_generator.py +8 -9
- cognite/neat/graph/loaders/__init__.py +5 -2
- cognite/neat/graph/loaders/_base.py +13 -5
- cognite/neat/graph/loaders/_rdf2asset.py +94 -20
- cognite/neat/graph/loaders/_rdf2dms.py +3 -16
- cognite/neat/graph/queries/_base.py +58 -5
- cognite/neat/graph/queries/_construct.py +17 -15
- cognite/neat/graph/queries/_shared.py +20 -6
- cognite/neat/graph/stores/_base.py +19 -10
- cognite/neat/graph/transformers/_rdfpath.py +7 -0
- cognite/neat/legacy/graph/extractors/_dexpi.py +0 -5
- cognite/neat/legacy/graph/stores/_base.py +24 -8
- cognite/neat/legacy/graph/stores/_graphdb_store.py +3 -2
- cognite/neat/legacy/graph/stores/_memory_store.py +3 -3
- cognite/neat/legacy/graph/stores/_oxigraph_store.py +8 -4
- cognite/neat/legacy/graph/stores/_rdf_to_graph.py +5 -3
- cognite/neat/legacy/graph/transformations/query_generator/sparql.py +48 -15
- cognite/neat/legacy/rules/importers/_graph2rules.py +34 -7
- cognite/neat/legacy/rules/models/raw_rules.py +18 -6
- cognite/neat/legacy/rules/models/rules.py +32 -12
- cognite/neat/rules/_shared.py +6 -1
- cognite/neat/rules/analysis/__init__.py +4 -4
- cognite/neat/rules/analysis/_asset.py +128 -0
- cognite/neat/rules/analysis/_base.py +385 -6
- cognite/neat/rules/analysis/_information.py +155 -0
- cognite/neat/rules/exporters/_base.py +4 -4
- cognite/neat/rules/exporters/_rules2dms.py +1 -1
- cognite/neat/rules/exporters/_rules2ontology.py +5 -5
- cognite/neat/rules/importers/_base.py +4 -4
- cognite/neat/rules/importers/_dtdl2rules/dtdl_converter.py +2 -8
- cognite/neat/rules/importers/_inference2rules.py +2 -2
- cognite/neat/rules/importers/_owl2rules/_owl2metadata.py +1 -1
- cognite/neat/rules/importers/_spreadsheet2rules.py +5 -5
- cognite/neat/rules/models/__init__.py +3 -3
- cognite/neat/rules/models/_base.py +10 -10
- cognite/neat/rules/models/asset/_rules.py +9 -10
- cognite/neat/rules/models/dms/_converter.py +4 -5
- cognite/neat/rules/models/dms/_rules.py +6 -3
- cognite/neat/rules/models/domain.py +5 -2
- cognite/neat/rules/models/entities.py +2 -9
- cognite/neat/rules/models/information/_converter.py +3 -3
- cognite/neat/rules/models/information/_rules.py +13 -11
- cognite/neat/rules/models/information/_rules_input.py +1 -2
- cognite/neat/rules/models/information/_validation.py +1 -1
- cognite/neat/utils/utils.py +54 -18
- cognite/neat/workflows/steps/lib/current/graph_store.py +28 -8
- cognite/neat/workflows/steps/lib/legacy/graph_extractor.py +129 -27
- cognite/neat/workflows/steps/lib/legacy/graph_store.py +4 -4
- {cognite_neat-0.85.12.dist-info → cognite_neat-0.87.0.dist-info}/METADATA +1 -1
- {cognite_neat-0.85.12.dist-info → cognite_neat-0.87.0.dist-info}/RECORD +56 -54
- cognite/neat/rules/analysis/_information_rules.py +0 -469
- {cognite_neat-0.85.12.dist-info → cognite_neat-0.87.0.dist-info}/LICENSE +0 -0
- {cognite_neat-0.85.12.dist-info → cognite_neat-0.87.0.dist-info}/WHEEL +0 -0
- {cognite_neat-0.85.12.dist-info → cognite_neat-0.87.0.dist-info}/entry_points.txt +0 -0
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
import math
|
|
2
2
|
import sys
|
|
3
3
|
from datetime import datetime
|
|
4
|
-
from typing import TYPE_CHECKING, Any, ClassVar, Literal
|
|
4
|
+
from typing import TYPE_CHECKING, Any, ClassVar, Literal
|
|
5
5
|
|
|
6
6
|
from pydantic import Field, field_serializer, field_validator, model_validator
|
|
7
7
|
from pydantic.main import IncEx
|
|
8
8
|
from rdflib import Namespace
|
|
9
9
|
|
|
10
|
-
from cognite.neat.constants import
|
|
10
|
+
from cognite.neat.constants import get_default_prefixes
|
|
11
11
|
from cognite.neat.issues import MultiValueError
|
|
12
12
|
from cognite.neat.rules import exceptions, issues
|
|
13
13
|
from cognite.neat.rules.models._base import (
|
|
@@ -38,10 +38,9 @@ from cognite.neat.rules.models.data_types import DataType
|
|
|
38
38
|
from cognite.neat.rules.models.domain import DomainRules
|
|
39
39
|
from cognite.neat.rules.models.entities import (
|
|
40
40
|
ClassEntity,
|
|
41
|
+
ClassEntityList,
|
|
41
42
|
EntityTypes,
|
|
42
43
|
MultiValueTypeInfo,
|
|
43
|
-
ParentClassEntity,
|
|
44
|
-
ParentEntityList,
|
|
45
44
|
ReferenceEntity,
|
|
46
45
|
Undefined,
|
|
47
46
|
UnknownEntity,
|
|
@@ -59,7 +58,7 @@ else:
|
|
|
59
58
|
|
|
60
59
|
|
|
61
60
|
class InformationMetadata(BaseMetadata):
|
|
62
|
-
role: ClassVar[RoleTypes] = RoleTypes.
|
|
61
|
+
role: ClassVar[RoleTypes] = RoleTypes.information
|
|
63
62
|
data_model_type: DataModelType = Field(DataModelType.enterprise, alias="dataModelType")
|
|
64
63
|
schema_: SchemaCompleteness = Field(SchemaCompleteness.partial, alias="schema")
|
|
65
64
|
extension: ExtensionCategoryType | None = ExtensionCategory.addition
|
|
@@ -114,6 +113,9 @@ class InformationMetadata(BaseMetadata):
|
|
|
114
113
|
def as_identifier(self) -> str:
|
|
115
114
|
return f"{self.prefix}:{self.name}"
|
|
116
115
|
|
|
116
|
+
def get_prefix(self) -> str:
|
|
117
|
+
return self.prefix
|
|
118
|
+
|
|
117
119
|
|
|
118
120
|
class InformationClass(SheetEntity):
|
|
119
121
|
"""
|
|
@@ -130,7 +132,7 @@ class InformationClass(SheetEntity):
|
|
|
130
132
|
class_: ClassEntity = Field(alias="Class")
|
|
131
133
|
name: str | None = Field(alias="Name", default=None)
|
|
132
134
|
description: str | None = Field(alias="Description", default=None)
|
|
133
|
-
parent:
|
|
135
|
+
parent: ClassEntityList | None = Field(alias="Parent Class", default=None)
|
|
134
136
|
reference: URLEntity | ReferenceEntity | None = Field(alias="Reference", default=None, union_mode="left_to_right")
|
|
135
137
|
match_type: MatchType | None = Field(alias="Match Type", default=None)
|
|
136
138
|
comment: str | None = Field(alias="Comment", default=None)
|
|
@@ -259,7 +261,7 @@ class InformationRules(BaseRules):
|
|
|
259
261
|
metadata: InformationMetadata = Field(alias="Metadata")
|
|
260
262
|
properties: SheetList[InformationProperty] = Field(alias="Properties")
|
|
261
263
|
classes: SheetList[InformationClass] = Field(alias="Classes")
|
|
262
|
-
prefixes: dict[str, Namespace] = Field(default_factory=
|
|
264
|
+
prefixes: dict[str, Namespace] = Field(default_factory=get_default_prefixes, alias="Prefixes")
|
|
263
265
|
last: "InformationRules | None" = Field(None, alias="Last")
|
|
264
266
|
reference: "InformationRules | None" = Field(None, alias="Reference")
|
|
265
267
|
|
|
@@ -268,7 +270,7 @@ class InformationRules(BaseRules):
|
|
|
268
270
|
if isinstance(values, dict):
|
|
269
271
|
return {key: Namespace(value) if isinstance(value, str) else value for key, value in values.items()}
|
|
270
272
|
elif values is None:
|
|
271
|
-
values =
|
|
273
|
+
values = get_default_prefixes()
|
|
272
274
|
return values
|
|
273
275
|
|
|
274
276
|
@model_validator(mode="after")
|
|
@@ -287,7 +289,7 @@ class InformationRules(BaseRules):
|
|
|
287
289
|
# update parent classes
|
|
288
290
|
for class_ in self.classes:
|
|
289
291
|
if class_.parent:
|
|
290
|
-
for parent in
|
|
292
|
+
for parent in class_.parent:
|
|
291
293
|
if not isinstance(parent.prefix, str):
|
|
292
294
|
parent.prefix = self.metadata.prefix
|
|
293
295
|
if class_.class_.prefix is Undefined:
|
|
@@ -348,7 +350,7 @@ class InformationRules(BaseRules):
|
|
|
348
350
|
|
|
349
351
|
return _InformationRulesConverter(self).as_asset_architect_rules()
|
|
350
352
|
|
|
351
|
-
def
|
|
353
|
+
def as_dms_rules(self) -> "DMSRules":
|
|
352
354
|
from ._converter import _InformationRulesConverter
|
|
353
355
|
|
|
354
|
-
return _InformationRulesConverter(self).
|
|
356
|
+
return _InformationRulesConverter(self).as_dms_rules()
|
|
@@ -15,7 +15,6 @@ from cognite.neat.rules.models.data_types import DataType
|
|
|
15
15
|
from cognite.neat.rules.models.entities import (
|
|
16
16
|
ClassEntity,
|
|
17
17
|
MultiValueTypeInfo,
|
|
18
|
-
ParentClassEntity,
|
|
19
18
|
Unknown,
|
|
20
19
|
UnknownEntity,
|
|
21
20
|
)
|
|
@@ -227,7 +226,7 @@ class InformationClassInput:
|
|
|
227
226
|
"Reference": self.reference,
|
|
228
227
|
"Match Type": self.match_type,
|
|
229
228
|
"Parent Class": (
|
|
230
|
-
[
|
|
229
|
+
[ClassEntity.load(parent, prefix=default_prefix) for parent in self.parent.split(",")]
|
|
231
230
|
if self.parent
|
|
232
231
|
else None
|
|
233
232
|
),
|
|
@@ -161,7 +161,7 @@ class InformationPostValidation:
|
|
|
161
161
|
class_subclass_pairs[class_.class_] = []
|
|
162
162
|
if class_.parent is None:
|
|
163
163
|
continue
|
|
164
|
-
class_subclass_pairs[class_.class_].extend(
|
|
164
|
+
class_subclass_pairs[class_.class_].extend(class_.parent)
|
|
165
165
|
|
|
166
166
|
return class_subclass_pairs
|
|
167
167
|
|
cognite/neat/utils/utils.py
CHANGED
|
@@ -7,20 +7,30 @@ from collections import Counter, OrderedDict
|
|
|
7
7
|
from collections.abc import Iterable
|
|
8
8
|
from datetime import datetime
|
|
9
9
|
from functools import wraps
|
|
10
|
-
from typing import TypeAlias, cast, overload
|
|
10
|
+
from typing import Literal, TypeAlias, cast, overload
|
|
11
11
|
|
|
12
12
|
import pandas as pd
|
|
13
13
|
from cognite.client import ClientConfig, CogniteClient
|
|
14
|
-
from cognite.client.credentials import
|
|
14
|
+
from cognite.client.credentials import (
|
|
15
|
+
CredentialProvider,
|
|
16
|
+
OAuthClientCredentials,
|
|
17
|
+
OAuthInteractive,
|
|
18
|
+
Token,
|
|
19
|
+
)
|
|
15
20
|
from cognite.client.exceptions import CogniteDuplicatedError, CogniteReadTimeout
|
|
16
21
|
from pydantic import HttpUrl, TypeAdapter, ValidationError
|
|
17
22
|
from pydantic_core import ErrorDetails
|
|
18
23
|
from pyparsing import Any
|
|
19
|
-
from rdflib import Literal
|
|
24
|
+
from rdflib import Literal as RdfLiteral
|
|
25
|
+
from rdflib import Namespace
|
|
20
26
|
from rdflib.term import URIRef
|
|
21
27
|
|
|
22
28
|
from cognite.neat import _version
|
|
23
|
-
from cognite.neat.utils.cdf import
|
|
29
|
+
from cognite.neat.utils.cdf import (
|
|
30
|
+
CogniteClientConfig,
|
|
31
|
+
InteractiveCogniteClient,
|
|
32
|
+
ServiceCogniteClient,
|
|
33
|
+
)
|
|
24
34
|
|
|
25
35
|
if sys.version_info >= (3, 11):
|
|
26
36
|
from datetime import UTC
|
|
@@ -30,12 +40,15 @@ else:
|
|
|
30
40
|
UTC = timezone.utc
|
|
31
41
|
|
|
32
42
|
|
|
33
|
-
Triple: TypeAlias = tuple[URIRef, URIRef,
|
|
43
|
+
Triple: TypeAlias = tuple[URIRef, URIRef, RdfLiteral | URIRef]
|
|
34
44
|
|
|
35
45
|
|
|
36
46
|
def get_cognite_client_from_config(config: ServiceCogniteClient) -> CogniteClient:
|
|
37
47
|
credentials = OAuthClientCredentials(
|
|
38
|
-
token_url=config.token_url,
|
|
48
|
+
token_url=config.token_url,
|
|
49
|
+
client_id=config.client_id,
|
|
50
|
+
client_secret=config.client_secret,
|
|
51
|
+
scopes=config.scopes,
|
|
39
52
|
)
|
|
40
53
|
|
|
41
54
|
return _get_cognite_client(config, credentials)
|
|
@@ -75,15 +88,25 @@ def _get_cognite_client(config: CogniteClientConfig, credentials: CredentialProv
|
|
|
75
88
|
|
|
76
89
|
|
|
77
90
|
@overload
|
|
78
|
-
def remove_namespace_from_uri(
|
|
91
|
+
def remove_namespace_from_uri(
|
|
92
|
+
*URI: URIRef | str,
|
|
93
|
+
special_separator: str = "#_",
|
|
94
|
+
validation: Literal["full", "prefix"] = "prefix",
|
|
95
|
+
) -> str: ...
|
|
79
96
|
|
|
80
97
|
|
|
81
98
|
@overload
|
|
82
|
-
def remove_namespace_from_uri(
|
|
99
|
+
def remove_namespace_from_uri(
|
|
100
|
+
*URI: tuple[URIRef | str, ...],
|
|
101
|
+
special_separator: str = "#_",
|
|
102
|
+
validation: Literal["full", "prefix"] = "prefix",
|
|
103
|
+
) -> tuple[str, ...]: ...
|
|
83
104
|
|
|
84
105
|
|
|
85
106
|
def remove_namespace_from_uri(
|
|
86
|
-
*URI: URIRef | str | tuple[URIRef | str, ...],
|
|
107
|
+
*URI: URIRef | str | tuple[URIRef | str, ...],
|
|
108
|
+
special_separator: str = "#_",
|
|
109
|
+
validation: Literal["full", "prefix"] = "prefix",
|
|
87
110
|
) -> tuple[str, ...] | str:
|
|
88
111
|
"""Removes namespace from URI
|
|
89
112
|
|
|
@@ -93,6 +116,9 @@ def remove_namespace_from_uri(
|
|
|
93
116
|
special_separator : str
|
|
94
117
|
Special separator to use instead of # or / if present in URI
|
|
95
118
|
Set by default to "#_" which covers special client use case
|
|
119
|
+
validation: str
|
|
120
|
+
Validation type to use for URI. If set to "full", URI will be validated using pydantic
|
|
121
|
+
If set to "prefix", only check if URI starts with http or https will be made
|
|
96
122
|
|
|
97
123
|
Returns
|
|
98
124
|
Entities id without namespace
|
|
@@ -114,11 +140,17 @@ def remove_namespace_from_uri(
|
|
|
114
140
|
|
|
115
141
|
output = []
|
|
116
142
|
for u in uris:
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
143
|
+
if validation == "full":
|
|
144
|
+
try:
|
|
145
|
+
_ = TypeAdapter(HttpUrl).validate_python(u)
|
|
146
|
+
output.append(u.split(special_separator if special_separator in u else "#" if "#" in u else "/")[-1])
|
|
147
|
+
except ValidationError:
|
|
148
|
+
output.append(str(u))
|
|
149
|
+
else:
|
|
150
|
+
if u.lower().startswith("http"):
|
|
151
|
+
output.append(u.split(special_separator if special_separator in u else "#" if "#" in u else "/")[-1])
|
|
152
|
+
else:
|
|
153
|
+
output.append(str(u))
|
|
122
154
|
|
|
123
155
|
return tuple(output) if len(output) > 1 else output[0]
|
|
124
156
|
|
|
@@ -154,8 +186,8 @@ def as_neat_compliant_uri(uri: URIRef) -> URIRef:
|
|
|
154
186
|
return URIRef(f"{namespace}{compliant_uri}")
|
|
155
187
|
|
|
156
188
|
|
|
157
|
-
def convert_rdflib_content(content:
|
|
158
|
-
if isinstance(content,
|
|
189
|
+
def convert_rdflib_content(content: RdfLiteral | URIRef | dict | list) -> Any:
|
|
190
|
+
if isinstance(content, RdfLiteral) or isinstance(content, URIRef):
|
|
159
191
|
return content.toPython()
|
|
160
192
|
elif isinstance(content, dict):
|
|
161
193
|
return {key: convert_rdflib_content(value) for key, value in content.items()}
|
|
@@ -192,7 +224,9 @@ def _traverse(hierarchy: dict, graph: dict, names: list[str]) -> dict:
|
|
|
192
224
|
|
|
193
225
|
|
|
194
226
|
def get_generation_order(
|
|
195
|
-
class_linkage: pd.DataFrame,
|
|
227
|
+
class_linkage: pd.DataFrame,
|
|
228
|
+
parent_col: str = "source_class",
|
|
229
|
+
child_col: str = "target_class",
|
|
196
230
|
) -> dict:
|
|
197
231
|
parent_child_list = class_linkage[[parent_col, child_col]].values.tolist()
|
|
198
232
|
# Build a directed graph and a list of all names that have no parent
|
|
@@ -318,7 +352,9 @@ def generate_exception_report(exceptions: list[dict] | list[ErrorDetails] | None
|
|
|
318
352
|
return report
|
|
319
353
|
|
|
320
354
|
|
|
321
|
-
def _order_expectations_by_type(
|
|
355
|
+
def _order_expectations_by_type(
|
|
356
|
+
exceptions: list[dict] | list[ErrorDetails],
|
|
357
|
+
) -> dict[str, list[str]]:
|
|
322
358
|
exception_dict: dict[str, list[str]] = {}
|
|
323
359
|
for exception in exceptions:
|
|
324
360
|
if not isinstance(exception["loc"], str) and isinstance(exception["loc"], Iterable):
|
|
@@ -2,11 +2,15 @@ import logging
|
|
|
2
2
|
from pathlib import Path
|
|
3
3
|
from typing import ClassVar, cast
|
|
4
4
|
|
|
5
|
-
from cognite.neat.constants import
|
|
5
|
+
from cognite.neat.constants import DEFAULT_NAMESPACE, get_default_prefixes
|
|
6
6
|
from cognite.neat.legacy.graph import stores
|
|
7
7
|
from cognite.neat.workflows._exceptions import StepNotInitialized
|
|
8
8
|
from cognite.neat.workflows.model import FlowMessage
|
|
9
|
-
from cognite.neat.workflows.steps.data_contracts import
|
|
9
|
+
from cognite.neat.workflows.steps.data_contracts import (
|
|
10
|
+
RulesData,
|
|
11
|
+
SolutionGraph,
|
|
12
|
+
SourceGraph,
|
|
13
|
+
)
|
|
10
14
|
from cognite.neat.workflows.steps.step_model import Configurable, Step
|
|
11
15
|
|
|
12
16
|
__all__ = ["GraphStoreConfiguration", "GraphStoreReset"]
|
|
@@ -40,9 +44,21 @@ class GraphStoreConfiguration(Step):
|
|
|
40
44
|
value="source-graph-store",
|
|
41
45
|
label="Local directory that is used as local graph store.Only for oxigraph, file store types",
|
|
42
46
|
),
|
|
43
|
-
Configurable(
|
|
44
|
-
|
|
45
|
-
|
|
47
|
+
Configurable(
|
|
48
|
+
name="Query URL",
|
|
49
|
+
value="",
|
|
50
|
+
label="Query URL for SPARQL endpoint. Only for SPARQL store type",
|
|
51
|
+
),
|
|
52
|
+
Configurable(
|
|
53
|
+
name="Update URL",
|
|
54
|
+
value="",
|
|
55
|
+
label="Update URL for SPARQL endpoint. Only for SPARQL store type",
|
|
56
|
+
),
|
|
57
|
+
Configurable(
|
|
58
|
+
name="GraphDB API root URL",
|
|
59
|
+
value="",
|
|
60
|
+
label="Root url for GraphDB. Only for graphdb",
|
|
61
|
+
),
|
|
46
62
|
Configurable(
|
|
47
63
|
name="Init procedure",
|
|
48
64
|
value="reset",
|
|
@@ -72,7 +88,7 @@ class GraphStoreConfiguration(Step):
|
|
|
72
88
|
graph_store = None
|
|
73
89
|
logging.info("Graph reset complete")
|
|
74
90
|
|
|
75
|
-
prefixes = rules_data.rules.prefixes if rules_data else
|
|
91
|
+
prefixes = rules_data.rules.prefixes if rules_data else get_default_prefixes()
|
|
76
92
|
|
|
77
93
|
if store_type == stores.OxiGraphStore.rdf_store_type and graph_store is not None:
|
|
78
94
|
# OXIGRAPH doesn't like to be initialized twice without a good reason
|
|
@@ -83,7 +99,7 @@ class GraphStoreConfiguration(Step):
|
|
|
83
99
|
except KeyError:
|
|
84
100
|
return FlowMessage(output_text="Invalid store type")
|
|
85
101
|
|
|
86
|
-
new_graph_store = store_cls(prefixes=prefixes, base_prefix="neat", namespace=
|
|
102
|
+
new_graph_store = store_cls(prefixes=prefixes, base_prefix="neat", namespace=DEFAULT_NAMESPACE)
|
|
87
103
|
new_graph_store.init_graph(
|
|
88
104
|
self.configs["Query URL"],
|
|
89
105
|
self.configs["Update URL"],
|
|
@@ -93,7 +109,11 @@ class GraphStoreConfiguration(Step):
|
|
|
93
109
|
|
|
94
110
|
return (
|
|
95
111
|
FlowMessage(output_text="Graph store configured successfully"),
|
|
96
|
-
|
|
112
|
+
(
|
|
113
|
+
SourceGraph(graph=new_graph_store)
|
|
114
|
+
if graph_name == "SourceGraph"
|
|
115
|
+
else SolutionGraph(graph=new_graph_store)
|
|
116
|
+
),
|
|
97
117
|
)
|
|
98
118
|
|
|
99
119
|
|
|
@@ -8,14 +8,20 @@ from typing import ClassVar, cast
|
|
|
8
8
|
|
|
9
9
|
from rdflib import RDF, XSD, Literal, Namespace, URIRef
|
|
10
10
|
|
|
11
|
-
from cognite.neat.constants import
|
|
11
|
+
from cognite.neat.constants import DEFAULT_NAMESPACE
|
|
12
12
|
from cognite.neat.legacy.graph import extractors
|
|
13
|
-
from cognite.neat.legacy.graph.extractors._mock_graph_generator import
|
|
13
|
+
from cognite.neat.legacy.graph.extractors._mock_graph_generator import (
|
|
14
|
+
generate_triples as generate_mock_triples,
|
|
15
|
+
)
|
|
14
16
|
from cognite.neat.legacy.rules.exporters._rules2triples import get_instances_as_triples
|
|
15
17
|
from cognite.neat.utils.utils import create_sha256_hash
|
|
16
18
|
from cognite.neat.workflows._exceptions import StepNotInitialized
|
|
17
19
|
from cognite.neat.workflows.model import FlowMessage, StepExecutionStatus
|
|
18
|
-
from cognite.neat.workflows.steps.data_contracts import
|
|
20
|
+
from cognite.neat.workflows.steps.data_contracts import (
|
|
21
|
+
RulesData,
|
|
22
|
+
SolutionGraph,
|
|
23
|
+
SourceGraph,
|
|
24
|
+
)
|
|
19
25
|
from cognite.neat.workflows.steps.step_model import Configurable, Step
|
|
20
26
|
|
|
21
27
|
__all__ = [
|
|
@@ -196,7 +202,10 @@ class ExtractGraphFromMockGraph(Step):
|
|
|
196
202
|
label="Target number of instances for each class",
|
|
197
203
|
),
|
|
198
204
|
Configurable(
|
|
199
|
-
name="graph_name",
|
|
205
|
+
name="graph_name",
|
|
206
|
+
value="solution",
|
|
207
|
+
label="The name of target graph.",
|
|
208
|
+
options=["source", "solution"],
|
|
200
209
|
),
|
|
201
210
|
]
|
|
202
211
|
|
|
@@ -225,7 +234,10 @@ class ExtractGraphFromMockGraph(Step):
|
|
|
225
234
|
try:
|
|
226
235
|
triples = generate_mock_triples(transformation_rules=transformation_rules.rules, class_count=class_count)
|
|
227
236
|
except Exception as e:
|
|
228
|
-
return FlowMessage(
|
|
237
|
+
return FlowMessage(
|
|
238
|
+
error_text=f"Error: {e}",
|
|
239
|
+
step_execution_status=StepExecutionStatus.ABORT_AND_FAIL,
|
|
240
|
+
)
|
|
229
241
|
|
|
230
242
|
logging.info("Adding mock triples to graph")
|
|
231
243
|
graph_store.graph.add_triples(triples, verbose=True) # type: ignore[arg-type]
|
|
@@ -243,7 +255,10 @@ class ExtractGraphFromRulesInstanceSheet(Step):
|
|
|
243
255
|
|
|
244
256
|
configurables: ClassVar[list[Configurable]] = [
|
|
245
257
|
Configurable(
|
|
246
|
-
name="graph_name",
|
|
258
|
+
name="graph_name",
|
|
259
|
+
value="solution",
|
|
260
|
+
label="The name of target graph.",
|
|
261
|
+
options=["source", "solution"],
|
|
247
262
|
),
|
|
248
263
|
]
|
|
249
264
|
|
|
@@ -264,7 +279,10 @@ class ExtractGraphFromRulesInstanceSheet(Step):
|
|
|
264
279
|
try:
|
|
265
280
|
graph_store.graph.add_triples(triples, verbose=True) # type: ignore[arg-type]
|
|
266
281
|
except Exception as e:
|
|
267
|
-
return FlowMessage(
|
|
282
|
+
return FlowMessage(
|
|
283
|
+
error_text=f"Error: {e}",
|
|
284
|
+
step_execution_status=StepExecutionStatus.ABORT_AND_FAIL,
|
|
285
|
+
)
|
|
268
286
|
|
|
269
287
|
return FlowMessage(output_text=output_text)
|
|
270
288
|
|
|
@@ -281,7 +299,7 @@ class ExtractGraphFromRulesDataModel(Step):
|
|
|
281
299
|
def run( # type: ignore[override, syntax]
|
|
282
300
|
self, transformation_rules: RulesData, source_graph: SourceGraph
|
|
283
301
|
) -> FlowMessage:
|
|
284
|
-
ns =
|
|
302
|
+
ns = DEFAULT_NAMESPACE
|
|
285
303
|
classes = transformation_rules.rules.classes
|
|
286
304
|
properties = transformation_rules.rules.properties
|
|
287
305
|
counter = 0
|
|
@@ -291,16 +309,29 @@ class ExtractGraphFromRulesDataModel(Step):
|
|
|
291
309
|
source_graph.graph.graph.add((rdf_instance_id, RDF.type, URIRef(ns + class_def.class_id)))
|
|
292
310
|
if class_def.parent_class:
|
|
293
311
|
source_graph.graph.graph.add(
|
|
294
|
-
(
|
|
312
|
+
(
|
|
313
|
+
rdf_instance_id,
|
|
314
|
+
URIRef(ns + "hasParent"),
|
|
315
|
+
URIRef(ns + "_" + cast(str, class_def.parent_class)),
|
|
316
|
+
)
|
|
295
317
|
)
|
|
296
318
|
counter += 1
|
|
297
319
|
|
|
298
320
|
for _property_name, property_def in properties.items():
|
|
299
321
|
rdf_instance_id = URIRef(ns + "_" + property_def.class_id)
|
|
300
322
|
source_graph.graph.graph.add(
|
|
301
|
-
(
|
|
323
|
+
(
|
|
324
|
+
rdf_instance_id,
|
|
325
|
+
URIRef(ns + property_def.property_id),
|
|
326
|
+
Literal(property_def.expected_value_type),
|
|
327
|
+
)
|
|
302
328
|
)
|
|
303
|
-
if property_def.expected_value_type.suffix not in (
|
|
329
|
+
if property_def.expected_value_type.suffix not in (
|
|
330
|
+
"string",
|
|
331
|
+
"integer",
|
|
332
|
+
"float",
|
|
333
|
+
"boolean",
|
|
334
|
+
):
|
|
304
335
|
source_graph.graph.graph.add(
|
|
305
336
|
(
|
|
306
337
|
rdf_instance_id,
|
|
@@ -324,10 +355,15 @@ class ExtractGraphFromJsonFile(Step):
|
|
|
324
355
|
version = "legacy"
|
|
325
356
|
configurables: ClassVar[list[Configurable]] = [
|
|
326
357
|
Configurable(
|
|
327
|
-
name="file_name",
|
|
358
|
+
name="file_name",
|
|
359
|
+
value="data_dump.json",
|
|
360
|
+
label="Full path to the file containing data dump in JSON format",
|
|
328
361
|
),
|
|
329
362
|
Configurable(
|
|
330
|
-
name="graph_name",
|
|
363
|
+
name="graph_name",
|
|
364
|
+
value="solution",
|
|
365
|
+
label="The name of target graph.",
|
|
366
|
+
options=["source", "solution"],
|
|
331
367
|
),
|
|
332
368
|
Configurable(
|
|
333
369
|
name="object_id_generation_method",
|
|
@@ -340,7 +376,12 @@ class ExtractGraphFromJsonFile(Step):
|
|
|
340
376
|
hash_of_json_element - takes a hash of the JSON element.Very generic method but \
|
|
341
377
|
can be slow working with big objects. \
|
|
342
378
|
uuid - generates a random UUID, the option produces unstables ids . ",
|
|
343
|
-
options=[
|
|
379
|
+
options=[
|
|
380
|
+
"source_object_properties",
|
|
381
|
+
"source_object_id_mapping",
|
|
382
|
+
"hash_of_json_element",
|
|
383
|
+
"uuid",
|
|
384
|
+
],
|
|
344
385
|
),
|
|
345
386
|
Configurable(
|
|
346
387
|
name="json_object_id_mapping",
|
|
@@ -359,10 +400,21 @@ class ExtractGraphFromJsonFile(Step):
|
|
|
359
400
|
value="http://purl.org/cognite/neat#",
|
|
360
401
|
label="Namespace to be used for the generated objects.",
|
|
361
402
|
),
|
|
362
|
-
Configurable(
|
|
403
|
+
Configurable(
|
|
404
|
+
name="namespace_prefix",
|
|
405
|
+
value="neat",
|
|
406
|
+
label="The prefix to be used for the namespace.",
|
|
407
|
+
),
|
|
363
408
|
]
|
|
364
409
|
|
|
365
|
-
def get_json_object_id(
|
|
410
|
+
def get_json_object_id(
|
|
411
|
+
self,
|
|
412
|
+
method,
|
|
413
|
+
object_name: str,
|
|
414
|
+
json_object: dict,
|
|
415
|
+
parent_object_id: str,
|
|
416
|
+
id_mapping: dict,
|
|
417
|
+
):
|
|
366
418
|
if method == "source_object_properties":
|
|
367
419
|
object_id = ""
|
|
368
420
|
if object_name in id_mapping:
|
|
@@ -385,7 +437,11 @@ class ExtractGraphFromJsonFile(Step):
|
|
|
385
437
|
# back to hashing
|
|
386
438
|
logging.debug(f"Object {object_name} doesn't have a valid id.Error : {e}")
|
|
387
439
|
object_id = self.get_json_object_id(
|
|
388
|
-
"hash_of_json_element",
|
|
440
|
+
"hash_of_json_element",
|
|
441
|
+
object_name,
|
|
442
|
+
json_object,
|
|
443
|
+
parent_object_id,
|
|
444
|
+
id_mapping,
|
|
389
445
|
)
|
|
390
446
|
else:
|
|
391
447
|
raise ValueError(
|
|
@@ -436,7 +492,11 @@ class ExtractGraphFromJsonFile(Step):
|
|
|
436
492
|
|
|
437
493
|
# Iterate through the JSON data and convert it to triples
|
|
438
494
|
def convert_json_to_triples(
|
|
439
|
-
data: dict,
|
|
495
|
+
data: dict,
|
|
496
|
+
parent_node: URIRef,
|
|
497
|
+
parent_object_id: str,
|
|
498
|
+
parent_node_path: str,
|
|
499
|
+
property_name=None,
|
|
440
500
|
):
|
|
441
501
|
nonlocal nodes_counter, property_counter
|
|
442
502
|
if isinstance(data, dict):
|
|
@@ -456,7 +516,13 @@ class ExtractGraphFromJsonFile(Step):
|
|
|
456
516
|
new_node = URIRef(ns + object_id)
|
|
457
517
|
graph.graph.add((new_node, RDF.type, URIRef(ns + property_name)))
|
|
458
518
|
if labels_mapping and property_name in labels_mapping:
|
|
459
|
-
graph.graph.add(
|
|
519
|
+
graph.graph.add(
|
|
520
|
+
(
|
|
521
|
+
new_node,
|
|
522
|
+
URIRef(ns + "label"),
|
|
523
|
+
Literal(data[labels_mapping[property_name]]),
|
|
524
|
+
)
|
|
525
|
+
)
|
|
460
526
|
else:
|
|
461
527
|
graph.graph.add((new_node, URIRef(ns + "label"), Literal(property_name)))
|
|
462
528
|
graph.graph.add((new_node, URIRef(ns + "parent"), parent_node))
|
|
@@ -470,7 +536,13 @@ class ExtractGraphFromJsonFile(Step):
|
|
|
470
536
|
convert_json_to_triples(value, parent_node, parent_object_id, parent_node_path, key)
|
|
471
537
|
else:
|
|
472
538
|
for item in data:
|
|
473
|
-
convert_json_to_triples(
|
|
539
|
+
convert_json_to_triples(
|
|
540
|
+
item,
|
|
541
|
+
parent_node,
|
|
542
|
+
parent_object_id,
|
|
543
|
+
parent_node_path,
|
|
544
|
+
property_name,
|
|
545
|
+
)
|
|
474
546
|
else:
|
|
475
547
|
# Convert scalar values to RDF literals
|
|
476
548
|
if isinstance(data, bool):
|
|
@@ -513,7 +585,10 @@ class ExtractGraphFromAvevaPiAssetFramework(Step):
|
|
|
513
585
|
containing data dump in XML format",
|
|
514
586
|
),
|
|
515
587
|
Configurable(
|
|
516
|
-
name="graph_name",
|
|
588
|
+
name="graph_name",
|
|
589
|
+
value="solution",
|
|
590
|
+
label="The name of target graph.",
|
|
591
|
+
options=["source", "solution"],
|
|
517
592
|
),
|
|
518
593
|
Configurable(
|
|
519
594
|
name="root_node_external_id",
|
|
@@ -535,7 +610,11 @@ class ExtractGraphFromAvevaPiAssetFramework(Step):
|
|
|
535
610
|
value="http://purl.org/cognite/neat#",
|
|
536
611
|
label="Namespace to be used for the generated objects.",
|
|
537
612
|
),
|
|
538
|
-
Configurable(
|
|
613
|
+
Configurable(
|
|
614
|
+
name="namespace_prefix",
|
|
615
|
+
value="neat",
|
|
616
|
+
label="The prefix to be used for the namespace.",
|
|
617
|
+
),
|
|
539
618
|
]
|
|
540
619
|
|
|
541
620
|
def add_root_asset_to_source_graph(self) -> str:
|
|
@@ -556,7 +635,10 @@ class ExtractGraphFromAvevaPiAssetFramework(Step):
|
|
|
556
635
|
if source_file := self.configs["file_name"]:
|
|
557
636
|
source_pi_dump = Path(self.data_store_path) / source_file
|
|
558
637
|
else:
|
|
559
|
-
return FlowMessage(
|
|
638
|
+
return FlowMessage(
|
|
639
|
+
output_text="No source file specified",
|
|
640
|
+
next_step_ids=["step_error_handler"],
|
|
641
|
+
)
|
|
560
642
|
|
|
561
643
|
# self.graph.bind
|
|
562
644
|
if self.configs["graph_name"] == "solution":
|
|
@@ -586,7 +668,11 @@ class ExtractGraphFromAvevaPiAssetFramework(Step):
|
|
|
586
668
|
self.graph_store.graph.add((rdf_instance_id, URIRef(self.ns + "Path"), Literal(new_element_path)))
|
|
587
669
|
if parent_element_id:
|
|
588
670
|
self.graph_store.graph.add(
|
|
589
|
-
(
|
|
671
|
+
(
|
|
672
|
+
rdf_instance_id,
|
|
673
|
+
URIRef(self.ns + "hasParent"),
|
|
674
|
+
URIRef(self.ns + parent_element_id),
|
|
675
|
+
)
|
|
590
676
|
)
|
|
591
677
|
for child in af_element:
|
|
592
678
|
if child.tag == "AFAttribute":
|
|
@@ -595,7 +681,13 @@ class ExtractGraphFromAvevaPiAssetFramework(Step):
|
|
|
595
681
|
pass
|
|
596
682
|
else:
|
|
597
683
|
try:
|
|
598
|
-
self.graph_store.graph.add(
|
|
684
|
+
self.graph_store.graph.add(
|
|
685
|
+
(
|
|
686
|
+
rdf_instance_id,
|
|
687
|
+
URIRef(self.ns + child.tag),
|
|
688
|
+
Literal(child.text),
|
|
689
|
+
)
|
|
690
|
+
)
|
|
599
691
|
except Exception as e:
|
|
600
692
|
logging.error(f"Error parsing AFAttribute {name} : {e}")
|
|
601
693
|
|
|
@@ -610,7 +702,11 @@ class ExtractGraphFromAvevaPiAssetFramework(Step):
|
|
|
610
702
|
self.graph_store.graph.add((rdf_instance_id, URIRef(self.ns + "Path"), Literal(new_element_path)))
|
|
611
703
|
if parent_element_id:
|
|
612
704
|
self.graph_store.graph.add(
|
|
613
|
-
(
|
|
705
|
+
(
|
|
706
|
+
rdf_instance_id,
|
|
707
|
+
URIRef(self.ns + "hasParent"),
|
|
708
|
+
URIRef(self.ns + parent_element_id),
|
|
709
|
+
)
|
|
614
710
|
)
|
|
615
711
|
|
|
616
712
|
for child in af_element:
|
|
@@ -624,7 +720,13 @@ class ExtractGraphFromAvevaPiAssetFramework(Step):
|
|
|
624
720
|
counter += 1
|
|
625
721
|
process_af_element(child, new_element_path, element_id)
|
|
626
722
|
else:
|
|
627
|
-
self.graph_store.graph.add(
|
|
723
|
+
self.graph_store.graph.add(
|
|
724
|
+
(
|
|
725
|
+
rdf_instance_id,
|
|
726
|
+
URIRef(self.ns + child.tag),
|
|
727
|
+
Literal(child.text),
|
|
728
|
+
)
|
|
729
|
+
)
|
|
628
730
|
|
|
629
731
|
if template:
|
|
630
732
|
self.graph_store.graph.add((rdf_instance_id, RDF.type, URIRef(self.ns + template)))
|