cognite-neat 0.88.0__py3-none-any.whl → 0.88.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_version.py +1 -1
- cognite/neat/app/api/routers/configuration.py +1 -1
- cognite/neat/app/ui/neat-app/build/asset-manifest.json +7 -7
- cognite/neat/app/ui/neat-app/build/index.html +1 -1
- cognite/neat/app/ui/neat-app/build/static/css/{main.38a62222.css → main.72e3d92e.css} +2 -2
- cognite/neat/app/ui/neat-app/build/static/css/main.72e3d92e.css.map +1 -0
- cognite/neat/app/ui/neat-app/build/static/js/main.5a52cf09.js +3 -0
- cognite/neat/app/ui/neat-app/build/static/js/{main.ec7f72e2.js.LICENSE.txt → main.5a52cf09.js.LICENSE.txt} +0 -9
- cognite/neat/app/ui/neat-app/build/static/js/main.5a52cf09.js.map +1 -0
- cognite/neat/config.py +44 -27
- cognite/neat/exceptions.py +8 -2
- cognite/neat/graph/extractors/_classic_cdf/_assets.py +21 -73
- cognite/neat/graph/extractors/_classic_cdf/_base.py +102 -0
- cognite/neat/graph/extractors/_classic_cdf/_events.py +46 -42
- cognite/neat/graph/extractors/_classic_cdf/_files.py +41 -45
- cognite/neat/graph/extractors/_classic_cdf/_labels.py +75 -52
- cognite/neat/graph/extractors/_classic_cdf/_relationships.py +49 -27
- cognite/neat/graph/extractors/_classic_cdf/_sequences.py +47 -50
- cognite/neat/graph/extractors/_classic_cdf/_timeseries.py +47 -49
- cognite/neat/graph/loaders/_base.py +4 -4
- cognite/neat/graph/loaders/_rdf2asset.py +12 -14
- cognite/neat/graph/loaders/_rdf2dms.py +14 -10
- cognite/neat/graph/queries/_base.py +22 -29
- cognite/neat/graph/queries/_shared.py +1 -1
- cognite/neat/graph/stores/_base.py +19 -11
- cognite/neat/graph/transformers/_rdfpath.py +3 -2
- cognite/neat/issues/__init__.py +16 -0
- cognite/neat/{issues.py → issues/_base.py} +78 -2
- cognite/neat/issues/errors/external.py +21 -0
- cognite/neat/issues/errors/properties.py +75 -0
- cognite/neat/issues/errors/resources.py +123 -0
- cognite/neat/issues/errors/schema.py +0 -0
- cognite/neat/{rules/issues → issues}/formatters.py +9 -9
- cognite/neat/issues/neat_warnings/__init__.py +2 -0
- cognite/neat/issues/neat_warnings/identifier.py +27 -0
- cognite/neat/issues/neat_warnings/models.py +22 -0
- cognite/neat/issues/neat_warnings/properties.py +77 -0
- cognite/neat/issues/neat_warnings/resources.py +125 -0
- cognite/neat/rules/exporters/_rules2dms.py +3 -2
- cognite/neat/rules/exporters/_rules2ontology.py +28 -20
- cognite/neat/rules/exporters/_validation.py +15 -21
- cognite/neat/rules/importers/__init__.py +7 -3
- cognite/neat/rules/importers/_base.py +3 -3
- cognite/neat/rules/importers/_dms2rules.py +39 -18
- cognite/neat/rules/importers/_dtdl2rules/dtdl_converter.py +44 -53
- cognite/neat/rules/importers/_dtdl2rules/dtdl_importer.py +6 -5
- cognite/neat/rules/importers/_rdf/__init__.py +0 -0
- cognite/neat/rules/importers/_rdf/_imf2rules/__init__.py +3 -0
- cognite/neat/rules/importers/_rdf/_imf2rules/_imf2classes.py +82 -0
- cognite/neat/rules/importers/_rdf/_imf2rules/_imf2metadata.py +34 -0
- cognite/neat/rules/importers/_rdf/_imf2rules/_imf2properties.py +123 -0
- cognite/neat/rules/importers/{_owl2rules/_owl2rules.py → _rdf/_imf2rules/_imf2rules.py} +15 -11
- cognite/neat/rules/importers/{_inference2rules.py → _rdf/_inference2rules.py} +1 -1
- cognite/neat/rules/importers/_rdf/_owl2rules/_owl2classes.py +57 -0
- cognite/neat/rules/importers/_rdf/_owl2rules/_owl2metadata.py +68 -0
- cognite/neat/rules/importers/_rdf/_owl2rules/_owl2properties.py +59 -0
- cognite/neat/rules/importers/_rdf/_owl2rules/_owl2rules.py +76 -0
- cognite/neat/rules/importers/_rdf/_shared.py +586 -0
- cognite/neat/rules/importers/_spreadsheet2rules.py +31 -28
- cognite/neat/rules/importers/_yaml2rules.py +2 -1
- cognite/neat/rules/issues/__init__.py +1 -5
- cognite/neat/rules/issues/base.py +2 -21
- cognite/neat/rules/issues/dms.py +20 -134
- cognite/neat/rules/issues/ontology.py +298 -0
- cognite/neat/rules/issues/spreadsheet.py +51 -3
- cognite/neat/rules/issues/tables.py +72 -0
- cognite/neat/rules/models/_rdfpath.py +4 -4
- cognite/neat/rules/models/_types/_field.py +14 -21
- cognite/neat/rules/models/asset/_validation.py +1 -1
- cognite/neat/rules/models/dms/_schema.py +53 -30
- cognite/neat/rules/models/dms/_validation.py +2 -2
- cognite/neat/rules/models/entities.py +3 -0
- cognite/neat/rules/models/information/_rules.py +5 -4
- cognite/neat/rules/models/information/_validation.py +1 -1
- cognite/neat/utils/rdf_.py +17 -9
- cognite/neat/utils/regex_patterns.py +52 -0
- cognite/neat/workflows/steps/lib/current/rules_importer.py +73 -1
- cognite/neat/workflows/steps/lib/current/rules_validator.py +19 -7
- {cognite_neat-0.88.0.dist-info → cognite_neat-0.88.2.dist-info}/METADATA +2 -6
- {cognite_neat-0.88.0.dist-info → cognite_neat-0.88.2.dist-info}/RECORD +85 -72
- cognite/neat/app/ui/neat-app/build/static/css/main.38a62222.css.map +0 -1
- cognite/neat/app/ui/neat-app/build/static/js/main.ec7f72e2.js +0 -3
- cognite/neat/app/ui/neat-app/build/static/js/main.ec7f72e2.js.map +0 -1
- cognite/neat/graph/issues/loader.py +0 -104
- cognite/neat/graph/stores/_oxrdflib.py +0 -247
- cognite/neat/rules/exceptions.py +0 -2972
- cognite/neat/rules/importers/_owl2rules/_owl2classes.py +0 -215
- cognite/neat/rules/importers/_owl2rules/_owl2metadata.py +0 -213
- cognite/neat/rules/importers/_owl2rules/_owl2properties.py +0 -203
- cognite/neat/rules/issues/importing.py +0 -408
- cognite/neat/rules/models/_types/_base.py +0 -16
- cognite/neat/workflows/examples/Export_Rules_to_Ontology/workflow.yaml +0 -152
- cognite/neat/workflows/examples/Extract_DEXPI_Graph_and_Export_Rules/workflow.yaml +0 -139
- cognite/neat/workflows/examples/Ontology_to_Data_Model/workflow.yaml +0 -116
- /cognite/neat/{graph/issues → issues/errors}/__init__.py +0 -0
- /cognite/neat/rules/importers/{_owl2rules → _rdf/_owl2rules}/__init__.py +0 -0
- {cognite_neat-0.88.0.dist-info → cognite_neat-0.88.2.dist-info}/LICENSE +0 -0
- {cognite_neat-0.88.0.dist-info → cognite_neat-0.88.2.dist-info}/WHEEL +0 -0
- {cognite_neat-0.88.0.dist-info → cognite_neat-0.88.2.dist-info}/entry_points.txt +0 -0
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
import re
|
|
2
1
|
import warnings
|
|
3
2
|
from typing import Literal, overload
|
|
4
3
|
|
|
5
4
|
from cognite.neat.exceptions import wrangle_warnings
|
|
6
|
-
from cognite.neat.
|
|
5
|
+
from cognite.neat.issues.neat_warnings.properties import PropertyRedefinedWarning
|
|
6
|
+
from cognite.neat.rules.issues.dms import EntityIDNotDMSCompliantWarning
|
|
7
7
|
from cognite.neat.rules.models import InformationRules
|
|
8
|
-
from cognite.neat.
|
|
8
|
+
from cognite.neat.utils.regex_patterns import DMS_PROPERTY_ID_COMPLIANCE_REGEX, PATTERNS, VIEW_ID_COMPLIANCE_REGEX
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
@overload
|
|
@@ -26,35 +26,30 @@ def are_entity_names_dms_compliant(
|
|
|
26
26
|
flag: bool = True
|
|
27
27
|
with warnings.catch_warnings(record=True) as validation_warnings:
|
|
28
28
|
for class_ in rules.classes:
|
|
29
|
-
if not
|
|
29
|
+
if not PATTERNS.view_id_compliance.match(class_.class_.suffix):
|
|
30
30
|
warnings.warn(
|
|
31
|
-
|
|
32
|
-
"Class", class_.class_.versioned_id, f"[Classes/Class/{class_.class_.versioned_id}]"
|
|
33
|
-
).message,
|
|
34
|
-
category=exceptions.EntityIDNotDMSCompliant,
|
|
31
|
+
EntityIDNotDMSCompliantWarning(class_.class_.versioned_id, "Class", VIEW_ID_COMPLIANCE_REGEX),
|
|
35
32
|
stacklevel=2,
|
|
36
33
|
)
|
|
37
34
|
flag = False
|
|
38
35
|
|
|
39
|
-
for
|
|
36
|
+
for _, property_ in enumerate(rules.properties):
|
|
40
37
|
# check class id which would resolve as view/container id
|
|
41
|
-
if not
|
|
38
|
+
if not PATTERNS.view_id_compliance.match(property_.class_.suffix):
|
|
42
39
|
warnings.warn(
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
40
|
+
EntityIDNotDMSCompliantWarning(
|
|
41
|
+
property_.class_.versioned_id,
|
|
42
|
+
"Class",
|
|
43
|
+
VIEW_ID_COMPLIANCE_REGEX,
|
|
44
|
+
),
|
|
47
45
|
stacklevel=2,
|
|
48
46
|
)
|
|
49
47
|
flag = False
|
|
50
48
|
|
|
51
49
|
# check property id which would resolve as view/container id
|
|
52
|
-
if not
|
|
50
|
+
if not PATTERNS.dms_property_id_compliance.match(property_.property_):
|
|
53
51
|
warnings.warn(
|
|
54
|
-
|
|
55
|
-
"Property", property_.property_, f"[Properties/Property/{row}]"
|
|
56
|
-
).message,
|
|
57
|
-
category=exceptions.EntityIDNotDMSCompliant,
|
|
52
|
+
EntityIDNotDMSCompliantWarning(property_.property_, "Property", DMS_PROPERTY_ID_COMPLIANCE_REGEX),
|
|
58
53
|
stacklevel=2,
|
|
59
54
|
)
|
|
60
55
|
flag = False
|
|
@@ -83,8 +78,7 @@ def are_properties_redefined(rules: InformationRules, return_report: bool = Fals
|
|
|
83
78
|
elif property_.class_ in analyzed_properties[property_.property_]:
|
|
84
79
|
flag = True
|
|
85
80
|
warnings.warn(
|
|
86
|
-
|
|
87
|
-
category=exceptions.EntityIDNotDMSCompliant,
|
|
81
|
+
PropertyRedefinedWarning[str](property_.class_.versioned_id, "Class", property_.property_),
|
|
88
82
|
stacklevel=2,
|
|
89
83
|
)
|
|
90
84
|
|
|
@@ -1,14 +1,16 @@
|
|
|
1
1
|
from ._base import BaseImporter
|
|
2
2
|
from ._dms2rules import DMSImporter
|
|
3
3
|
from ._dtdl2rules import DTDLImporter
|
|
4
|
-
from .
|
|
5
|
-
from .
|
|
4
|
+
from ._rdf._imf2rules import IMFImporter
|
|
5
|
+
from ._rdf._inference2rules import InferenceImporter
|
|
6
|
+
from ._rdf._owl2rules import OWLImporter
|
|
6
7
|
from ._spreadsheet2rules import ExcelImporter, GoogleSheetImporter
|
|
7
8
|
from ._yaml2rules import YAMLImporter
|
|
8
9
|
|
|
9
10
|
__all__ = [
|
|
10
11
|
"BaseImporter",
|
|
11
12
|
"OWLImporter",
|
|
13
|
+
"IMFImporter",
|
|
12
14
|
"DMSImporter",
|
|
13
15
|
"ExcelImporter",
|
|
14
16
|
"GoogleSheetImporter",
|
|
@@ -25,7 +27,9 @@ def _repr_html_() -> str:
|
|
|
25
27
|
[
|
|
26
28
|
{
|
|
27
29
|
"Importer": name,
|
|
28
|
-
"Description":
|
|
30
|
+
"Description": (
|
|
31
|
+
globals()[name].__doc__.strip().split("\n")[0] if globals()[name].__doc__ else "Missing"
|
|
32
|
+
),
|
|
29
33
|
}
|
|
30
34
|
for name in __all__
|
|
31
35
|
if name != "BaseImporter"
|
|
@@ -9,9 +9,9 @@ from typing import Any, Literal, overload
|
|
|
9
9
|
from pydantic import ValidationError
|
|
10
10
|
from rdflib import Namespace
|
|
11
11
|
|
|
12
|
+
from cognite.neat.issues import IssueList, NeatError, NeatWarning
|
|
12
13
|
from cognite.neat.rules._shared import Rules
|
|
13
14
|
from cognite.neat.rules.issues.base import (
|
|
14
|
-
IssueList,
|
|
15
15
|
NeatValidationError,
|
|
16
16
|
ValidationWarning,
|
|
17
17
|
)
|
|
@@ -103,8 +103,8 @@ class _FutureResult:
|
|
|
103
103
|
@contextmanager
|
|
104
104
|
def _handle_issues(
|
|
105
105
|
issues: IssueList,
|
|
106
|
-
error_cls: type[
|
|
107
|
-
warning_cls: type[
|
|
106
|
+
error_cls: type[NeatError] = NeatValidationError,
|
|
107
|
+
warning_cls: type[NeatWarning] = ValidationWarning,
|
|
108
108
|
error_args: dict[str, Any] | None = None,
|
|
109
109
|
) -> Iterator[_FutureResult]:
|
|
110
110
|
"""This is an internal help function to handle issues and warnings.
|
|
@@ -18,9 +18,15 @@ from cognite.client.data_classes.data_modeling.views import (
|
|
|
18
18
|
)
|
|
19
19
|
from cognite.client.utils import ms_to_datetime
|
|
20
20
|
|
|
21
|
+
from cognite.neat.issues import IssueList, NeatIssue
|
|
22
|
+
from cognite.neat.issues.errors.resources import ResourceNotFoundError
|
|
23
|
+
from cognite.neat.issues.neat_warnings.properties import (
|
|
24
|
+
PropertyTypeNotSupportedWarning,
|
|
25
|
+
ReferredPropertyNotFoundWarning,
|
|
26
|
+
)
|
|
27
|
+
from cognite.neat.issues.neat_warnings.resources import ReferredResourceNotFoundWarning
|
|
21
28
|
from cognite.neat.rules import issues
|
|
22
29
|
from cognite.neat.rules.importers._base import BaseImporter, Rules, _handle_issues
|
|
23
|
-
from cognite.neat.rules.issues import IssueList, ValidationIssue
|
|
24
30
|
from cognite.neat.rules.models import (
|
|
25
31
|
DataModelType,
|
|
26
32
|
DMSRules,
|
|
@@ -60,7 +66,7 @@ class DMSImporter(BaseImporter):
|
|
|
60
66
|
def __init__(
|
|
61
67
|
self,
|
|
62
68
|
schema: DMSSchema,
|
|
63
|
-
read_issues: Sequence[
|
|
69
|
+
read_issues: Sequence[NeatIssue] | None = None,
|
|
64
70
|
metadata: DMSMetadata | None = None,
|
|
65
71
|
ref_metadata: DMSMetadata | None = None,
|
|
66
72
|
):
|
|
@@ -100,14 +106,28 @@ class DMSImporter(BaseImporter):
|
|
|
100
106
|
|
|
101
107
|
user_models = cls._find_model_in_list(data_models, data_model_id)
|
|
102
108
|
if len(user_models) == 0:
|
|
103
|
-
return cls(
|
|
109
|
+
return cls(
|
|
110
|
+
DMSSchema(),
|
|
111
|
+
[
|
|
112
|
+
ResourceNotFoundError[dm.DataModelId](
|
|
113
|
+
dm.DataModelId.load(reference_model_id), # type: ignore[arg-type]
|
|
114
|
+
"DataModel",
|
|
115
|
+
"Data Model is missing in CDF",
|
|
116
|
+
)
|
|
117
|
+
],
|
|
118
|
+
)
|
|
104
119
|
user_model = user_models.latest_version()
|
|
105
120
|
|
|
106
121
|
if reference_model_id:
|
|
107
122
|
ref_models = cls._find_model_in_list(data_models, reference_model_id)
|
|
108
123
|
if len(ref_models) == 0:
|
|
109
124
|
return cls(
|
|
110
|
-
DMSSchema(),
|
|
125
|
+
DMSSchema(),
|
|
126
|
+
[
|
|
127
|
+
ResourceNotFoundError[dm.DataModelId](
|
|
128
|
+
dm.DataModelId.load(reference_model_id), "DataModel", "Data Model is missing in CDF"
|
|
129
|
+
)
|
|
130
|
+
],
|
|
111
131
|
)
|
|
112
132
|
ref_model: dm.DataModel[dm.View] | None = ref_models.latest_version()
|
|
113
133
|
else:
|
|
@@ -200,7 +220,7 @@ class DMSImporter(BaseImporter):
|
|
|
200
220
|
return self._return_or_raise(self.issue_list, errors)
|
|
201
221
|
|
|
202
222
|
if not self.root_schema.data_model:
|
|
203
|
-
self.issue_list.append(
|
|
223
|
+
self.issue_list.append(ResourceNotFoundError[str]("Unknown", "DataModel", "Identifier is missing"))
|
|
204
224
|
return self._return_or_raise(self.issue_list, errors)
|
|
205
225
|
model = self.root_schema.data_model
|
|
206
226
|
with _handle_issues(
|
|
@@ -301,10 +321,11 @@ class DMSImporter(BaseImporter):
|
|
|
301
321
|
) -> DMSProperty | None:
|
|
302
322
|
if isinstance(prop, dm.MappedPropertyApply) and prop.container not in self._all_containers_by_id:
|
|
303
323
|
self.issue_list.append(
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
324
|
+
ReferredResourceNotFoundWarning[dm.ContainerId, dm.PropertyId](
|
|
325
|
+
dm.ContainerId.load(prop.container),
|
|
326
|
+
"Container",
|
|
327
|
+
view_entity.to_property_id(prop_id),
|
|
328
|
+
"View Property",
|
|
308
329
|
)
|
|
309
330
|
)
|
|
310
331
|
return None
|
|
@@ -313,11 +334,9 @@ class DMSImporter(BaseImporter):
|
|
|
313
334
|
and prop.container_property_identifier not in self._all_containers_by_id[prop.container].properties
|
|
314
335
|
):
|
|
315
336
|
self.issue_list.append(
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
container_id=str(ContainerEntity.from_id(prop.container)),
|
|
320
|
-
)
|
|
337
|
+
ReferredPropertyNotFoundWarning[dm.ContainerId, dm.ViewId](
|
|
338
|
+
prop.container, "Container", view_entity.as_id(), "View", prop_id
|
|
339
|
+
),
|
|
321
340
|
)
|
|
322
341
|
return None
|
|
323
342
|
if not isinstance(
|
|
@@ -329,7 +348,7 @@ class DMSImporter(BaseImporter):
|
|
|
329
348
|
| MultiReverseDirectRelationApply,
|
|
330
349
|
):
|
|
331
350
|
self.issue_list.append(
|
|
332
|
-
|
|
351
|
+
PropertyTypeNotSupportedWarning[dm.ViewId](view_entity.as_id(), "View", prop_id, type(prop).__name__)
|
|
333
352
|
)
|
|
334
353
|
return None
|
|
335
354
|
|
|
@@ -394,7 +413,9 @@ class DMSImporter(BaseImporter):
|
|
|
394
413
|
else:
|
|
395
414
|
return DataType.load(container_prop.type._type)
|
|
396
415
|
else:
|
|
397
|
-
self.issue_list.append(
|
|
416
|
+
self.issue_list.append(
|
|
417
|
+
PropertyTypeNotSupportedWarning[dm.ViewId](view_entity.as_id(), "View", prop_id, type(prop).__name__)
|
|
418
|
+
)
|
|
398
419
|
return None
|
|
399
420
|
|
|
400
421
|
def _get_nullable(self, prop: ViewPropertyApply) -> bool | None:
|
|
@@ -453,8 +474,8 @@ class DMSImporter(BaseImporter):
|
|
|
453
474
|
continue
|
|
454
475
|
else:
|
|
455
476
|
self.issue_list.append(
|
|
456
|
-
|
|
457
|
-
|
|
477
|
+
PropertyTypeNotSupportedWarning[dm.ContainerId](
|
|
478
|
+
prop.container, "Container", prop_id, type(constraint_obj).__name__
|
|
458
479
|
)
|
|
459
480
|
)
|
|
460
481
|
return unique_constraints or None
|
|
@@ -1,14 +1,18 @@
|
|
|
1
1
|
from collections import Counter
|
|
2
2
|
from collections.abc import Callable, Sequence
|
|
3
3
|
|
|
4
|
-
|
|
5
|
-
from cognite.neat.
|
|
4
|
+
from cognite.neat.issues import IssueList, NeatIssue
|
|
5
|
+
from cognite.neat.issues.errors.properties import PropertyTypeNotSupportedError
|
|
6
|
+
from cognite.neat.issues.errors.resources import MissingIdentifierError, ResourceNotFoundError
|
|
7
|
+
from cognite.neat.issues.neat_warnings.properties import PropertyTypeNotSupportedWarning
|
|
8
|
+
from cognite.neat.issues.neat_warnings.resources import ResourceTypeNotSupportedWarning
|
|
6
9
|
from cognite.neat.rules.importers._dtdl2rules.spec import (
|
|
7
10
|
DTMI,
|
|
8
11
|
Command,
|
|
9
12
|
CommandV2,
|
|
10
13
|
Component,
|
|
11
14
|
DTDLBase,
|
|
15
|
+
DTDLBaseWithName,
|
|
12
16
|
Enum,
|
|
13
17
|
Interface,
|
|
14
18
|
Object,
|
|
@@ -19,15 +23,14 @@ from cognite.neat.rules.importers._dtdl2rules.spec import (
|
|
|
19
23
|
Telemetry,
|
|
20
24
|
TelemetryV2,
|
|
21
25
|
)
|
|
22
|
-
from cognite.neat.rules.issues import IssueList, ValidationIssue
|
|
23
26
|
from cognite.neat.rules.models.data_types import _DATA_TYPE_BY_NAME, DataType, Json, String
|
|
24
27
|
from cognite.neat.rules.models.entities import ClassEntity
|
|
25
28
|
from cognite.neat.rules.models.information import InformationClass, InformationProperty
|
|
26
29
|
|
|
27
30
|
|
|
28
31
|
class _DTDLConverter:
|
|
29
|
-
def __init__(self, issues: list[
|
|
30
|
-
self.issues
|
|
32
|
+
def __init__(self, issues: list[NeatIssue] | None = None) -> None:
|
|
33
|
+
self.issues = IssueList(issues or [])
|
|
31
34
|
self.properties: list[InformationProperty] = []
|
|
32
35
|
self.classes: list[InformationClass] = []
|
|
33
36
|
self._item_by_id: dict[DTMI, DTDLBase] = {}
|
|
@@ -75,11 +78,10 @@ class _DTDLConverter:
|
|
|
75
78
|
convert_method(item, parent)
|
|
76
79
|
else:
|
|
77
80
|
self.issues.append(
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
)
|
|
81
|
+
ResourceTypeNotSupportedWarning[str](
|
|
82
|
+
item.id_.model_dump() if item.id_ else item.display_name or "missing",
|
|
83
|
+
item.type,
|
|
84
|
+
),
|
|
83
85
|
)
|
|
84
86
|
|
|
85
87
|
def convert_interface(self, item: Interface, _: str | None) -> None:
|
|
@@ -94,11 +96,11 @@ class _DTDLConverter:
|
|
|
94
96
|
for sub_item_or_id in item.contents or []:
|
|
95
97
|
if isinstance(sub_item_or_id, DTMI) and sub_item_or_id not in self._item_by_id:
|
|
96
98
|
self.issues.append(
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
99
|
+
PropertyTypeNotSupportedWarning(
|
|
100
|
+
item.id_.model_dump() or item.display_name or "missing",
|
|
101
|
+
item.type,
|
|
102
|
+
sub_item_or_id.path[-1],
|
|
103
|
+
".".join(sub_item_or_id.path),
|
|
102
104
|
)
|
|
103
105
|
)
|
|
104
106
|
elif isinstance(sub_item_or_id, DTMI):
|
|
@@ -130,12 +132,10 @@ class _DTDLConverter:
|
|
|
130
132
|
)
|
|
131
133
|
self.properties.append(prop)
|
|
132
134
|
|
|
133
|
-
def _missing_parent_warning(self, item):
|
|
135
|
+
def _missing_parent_warning(self, item: DTDLBaseWithName):
|
|
134
136
|
self.issues.append(
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
instance_name=item.display_name,
|
|
138
|
-
instance_id=item.id_.model_dump() if item.id_ else None,
|
|
137
|
+
ResourceNotFoundError[str](
|
|
138
|
+
(item.id_.model_dump() if item.id_ else item.display_name) or "missing", item.type, "parent missing"
|
|
139
139
|
)
|
|
140
140
|
)
|
|
141
141
|
|
|
@@ -151,22 +151,15 @@ class _DTDLConverter:
|
|
|
151
151
|
return None
|
|
152
152
|
if item.request is None:
|
|
153
153
|
self.issues.append(
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
instance_id=item.id_.model_dump() if item.id_ else None,
|
|
159
|
-
)
|
|
154
|
+
ResourceTypeNotSupportedWarning[str](
|
|
155
|
+
item.id_.model_dump() if item.id_ else item.display_name or "missing",
|
|
156
|
+
f"{item.type}.request",
|
|
157
|
+
),
|
|
160
158
|
)
|
|
161
159
|
return None
|
|
162
160
|
if item.response is not None:
|
|
163
161
|
# Currently, we do not know how to handle response
|
|
164
|
-
self.issues.append(
|
|
165
|
-
issues.importing.IgnoredComponentWarning(
|
|
166
|
-
identifier=f"{parent}.response",
|
|
167
|
-
reason="Neat does not have a concept of response for commands. This will be ignored.",
|
|
168
|
-
)
|
|
169
|
-
)
|
|
162
|
+
self.issues.append(ResourceTypeNotSupportedWarning[str](f"{parent}.response", "Command.Response"))
|
|
170
163
|
value_type = self.schema_to_value_type(item.request.schema_, item)
|
|
171
164
|
if value_type is None:
|
|
172
165
|
return
|
|
@@ -213,10 +206,9 @@ class _DTDLConverter:
|
|
|
213
206
|
else:
|
|
214
207
|
# Falling back to json
|
|
215
208
|
self.issues.append(
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
instance_id=item.target.model_dump(),
|
|
209
|
+
MissingIdentifierError(
|
|
210
|
+
"Unknown",
|
|
211
|
+
item.target.model_dump(),
|
|
220
212
|
)
|
|
221
213
|
)
|
|
222
214
|
value_type = Json()
|
|
@@ -239,9 +231,9 @@ class _DTDLConverter:
|
|
|
239
231
|
def convert_object(self, item: Object, _: str | None) -> None:
|
|
240
232
|
if item.id_ is None:
|
|
241
233
|
self.issues.append(
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
234
|
+
MissingIdentifierError(
|
|
235
|
+
resource_type=item.type,
|
|
236
|
+
name=item.display_name,
|
|
245
237
|
)
|
|
246
238
|
)
|
|
247
239
|
return None
|
|
@@ -280,21 +272,20 @@ class _DTDLConverter:
|
|
|
280
272
|
return _DATA_TYPE_BY_NAME[input_type.casefold()]()
|
|
281
273
|
elif isinstance(input_type, str):
|
|
282
274
|
self.issues.append(
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
instance_id=item.id_.model_dump() if item.id_ else None,
|
|
275
|
+
PropertyTypeNotSupportedError[str](
|
|
276
|
+
(item.id_.model_dump() if item.id_ else item.display_name) or "missing",
|
|
277
|
+
item.type,
|
|
278
|
+
"schema",
|
|
279
|
+
input_type,
|
|
289
280
|
)
|
|
290
281
|
)
|
|
291
282
|
return None
|
|
292
283
|
elif isinstance(input_type, Object | Interface):
|
|
293
284
|
if input_type.id_ is None:
|
|
294
285
|
self.issues.append(
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
286
|
+
MissingIdentifierError(
|
|
287
|
+
input_type.type,
|
|
288
|
+
input_type.display_name,
|
|
298
289
|
)
|
|
299
290
|
)
|
|
300
291
|
return Json()
|
|
@@ -304,11 +295,11 @@ class _DTDLConverter:
|
|
|
304
295
|
return ClassEntity.load(input_type.id_.as_class_id())
|
|
305
296
|
else:
|
|
306
297
|
self.issues.append(
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
298
|
+
PropertyTypeNotSupportedWarning(
|
|
299
|
+
item.id_.model_dump() if item.id_ else item.display_name or "missing",
|
|
300
|
+
item.type,
|
|
301
|
+
"schema",
|
|
302
|
+
input_type.type if input_type else "missing",
|
|
312
303
|
)
|
|
313
304
|
)
|
|
314
305
|
return None
|
|
@@ -6,12 +6,13 @@ from typing import Literal, overload
|
|
|
6
6
|
|
|
7
7
|
from pydantic import ValidationError
|
|
8
8
|
|
|
9
|
+
from cognite.neat.issues import IssueList, NeatIssue
|
|
9
10
|
from cognite.neat.rules import issues
|
|
10
11
|
from cognite.neat.rules._shared import Rules
|
|
11
12
|
from cognite.neat.rules.importers._base import BaseImporter, _handle_issues
|
|
12
13
|
from cognite.neat.rules.importers._dtdl2rules.dtdl_converter import _DTDLConverter
|
|
13
14
|
from cognite.neat.rules.importers._dtdl2rules.spec import DTDL_CLS_BY_TYPE_BY_SPEC, DTDLBase, Interface
|
|
14
|
-
from cognite.neat.rules.issues import
|
|
15
|
+
from cognite.neat.rules.issues import ValidationIssue
|
|
15
16
|
from cognite.neat.rules.models import InformationRules, RoleTypes, SchemaCompleteness, SheetList
|
|
16
17
|
from cognite.neat.rules.models.information import InformationClass, InformationProperty
|
|
17
18
|
from cognite.neat.utils.text import to_pascal
|
|
@@ -36,7 +37,7 @@ class DTDLImporter(BaseImporter):
|
|
|
36
37
|
self,
|
|
37
38
|
items: Sequence[DTDLBase],
|
|
38
39
|
title: str | None = None,
|
|
39
|
-
read_issues: list[
|
|
40
|
+
read_issues: list[NeatIssue] | None = None,
|
|
40
41
|
schema: SchemaCompleteness = SchemaCompleteness.partial,
|
|
41
42
|
) -> None:
|
|
42
43
|
self._items = items
|
|
@@ -94,10 +95,10 @@ class DTDLImporter(BaseImporter):
|
|
|
94
95
|
@classmethod
|
|
95
96
|
def from_directory(cls, directory: Path) -> "DTDLImporter":
|
|
96
97
|
items: list[DTDLBase] = []
|
|
97
|
-
issues: list[
|
|
98
|
+
issues: list[NeatIssue] = []
|
|
98
99
|
for filepath in directory.glob("**/*.json"):
|
|
99
100
|
for item in cls._from_file_content(filepath.read_text(), filepath):
|
|
100
|
-
if isinstance(item,
|
|
101
|
+
if isinstance(item, NeatIssue):
|
|
101
102
|
issues.append(item)
|
|
102
103
|
else:
|
|
103
104
|
items.append(item)
|
|
@@ -106,7 +107,7 @@ class DTDLImporter(BaseImporter):
|
|
|
106
107
|
@classmethod
|
|
107
108
|
def from_zip(cls, zip_file: Path) -> "DTDLImporter":
|
|
108
109
|
items: list[DTDLBase] = []
|
|
109
|
-
issues: list[
|
|
110
|
+
issues: list[NeatIssue] = []
|
|
110
111
|
with zipfile.ZipFile(zip_file) as z:
|
|
111
112
|
for filepath in z.namelist():
|
|
112
113
|
if filepath.endswith(".json"):
|
|
File without changes
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
from typing import cast
|
|
2
|
+
|
|
3
|
+
from rdflib import Graph
|
|
4
|
+
|
|
5
|
+
from cognite.neat.rules.importers._rdf._shared import (
|
|
6
|
+
clean_up_classes,
|
|
7
|
+
make_classes_compliant,
|
|
8
|
+
parse_raw_classes_dataframe,
|
|
9
|
+
)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def parse_imf_to_classes(graph: Graph, language: str = "en") -> list[dict]:
|
|
13
|
+
"""Parse IMF elements from RDF-graph and extract classes to pandas dataframe.
|
|
14
|
+
|
|
15
|
+
Args:
|
|
16
|
+
graph: Graph containing imf elements
|
|
17
|
+
language: Language to use for parsing, by default "en"
|
|
18
|
+
|
|
19
|
+
Returns:
|
|
20
|
+
Dataframe containing imf elements
|
|
21
|
+
|
|
22
|
+
!!! note "IMF Compliance"
|
|
23
|
+
The IMF elements are expressed in RDF, primarily using SHACL and OWL. To ensure
|
|
24
|
+
that the resulting classes are compliant with CDF, similar validation checks as
|
|
25
|
+
in the OWL ontology importer are applied.
|
|
26
|
+
|
|
27
|
+
For the IMF-types more of the compliance logic is placed directly in the SPARQL
|
|
28
|
+
query. Among these are the creation of class name not starting with a number,
|
|
29
|
+
and ensuring that all classes have a parent.
|
|
30
|
+
|
|
31
|
+
IMF-attributes are considered both classes and properties. This kind of punning
|
|
32
|
+
is necessary to capture additional information carried by attributes. They carry,
|
|
33
|
+
among other things, a set of relationsships to reference terms, units of measure,
|
|
34
|
+
and qualifiers that together make up the meaning of the attribute.
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
query = """
|
|
38
|
+
SELECT ?class ?name ?description ?parentClass ?reference ?match ?comment
|
|
39
|
+
WHERE {
|
|
40
|
+
#Finding IMF - elements
|
|
41
|
+
VALUES ?type { imf:BlockType imf:TerminalType imf:AttributeType }
|
|
42
|
+
?imfClass a ?type .
|
|
43
|
+
OPTIONAL {?imfClass rdfs:subClassOf ?parent }.
|
|
44
|
+
OPTIONAL {?imfClass rdfs:label | skos:prefLabel ?name }.
|
|
45
|
+
OPTIONAL {?imfClass rdfs:comment | skos:description ?description} .
|
|
46
|
+
|
|
47
|
+
# Finding the last segment of the class IRI
|
|
48
|
+
BIND(STR(?imfClass) AS ?classString)
|
|
49
|
+
BIND(REPLACE(?classString, "^.*[/#]([^/#]*)$", "$1") AS ?classSegment)
|
|
50
|
+
BIND(IF(CONTAINS(?classString, "imf/"), CONCAT("IMF_", ?classSegment) , ?classSegment) AS ?class)
|
|
51
|
+
|
|
52
|
+
# Add imf:Attribute as parent class
|
|
53
|
+
BIND(IF(!bound(?parent) && ?type = imf:AttributeType, imf:Attribute, ?parent) AS ?parentClass)
|
|
54
|
+
|
|
55
|
+
# Rebind the IRI of the IMF-type to the ?reference variable to align with dataframe column headers
|
|
56
|
+
# This is solely for readability, the ?imfClass could have been returnered directly instead of ?reference
|
|
57
|
+
BIND(?imfClass AS ?reference)
|
|
58
|
+
|
|
59
|
+
FILTER (!isBlank(?class))
|
|
60
|
+
FILTER (!bound(?parentClass) || !isBlank(?parentClass))
|
|
61
|
+
FILTER (!bound(?name) || LANG(?name) = "" || LANGMATCHES(LANG(?name), "en"))
|
|
62
|
+
FILTER (!bound(?description) || LANG(?description) = "" || LANGMATCHES(LANG(?description), "en"))
|
|
63
|
+
}
|
|
64
|
+
"""
|
|
65
|
+
|
|
66
|
+
# create raw dataframe
|
|
67
|
+
raw_df = parse_raw_classes_dataframe(cast(list[tuple], list(graph.query(query.replace("en", language)))))
|
|
68
|
+
if raw_df.empty:
|
|
69
|
+
return []
|
|
70
|
+
|
|
71
|
+
# group values and clean up
|
|
72
|
+
processed_df = clean_up_classes(raw_df)
|
|
73
|
+
|
|
74
|
+
# make compliant
|
|
75
|
+
processed_df = make_classes_compliant(processed_df, importer="IMF")
|
|
76
|
+
|
|
77
|
+
# Make Parent Class list elements into string joined with comma
|
|
78
|
+
processed_df["Parent Class"] = processed_df["Parent Class"].apply(
|
|
79
|
+
lambda x: ", ".join(x) if isinstance(x, list) and x else None
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
return processed_df.dropna(axis=0, how="all").replace(float("nan"), None).to_dict(orient="records")
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
from rdflib import Namespace
|
|
2
|
+
|
|
3
|
+
from cognite.neat.rules.importers._rdf._shared import make_metadata_compliant
|
|
4
|
+
from cognite.neat.rules.models import RoleTypes, SchemaCompleteness
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def parse_imf_metadata() -> dict:
|
|
8
|
+
"""Provide hardcoded IMF metadata to dict.
|
|
9
|
+
|
|
10
|
+
Returns:
|
|
11
|
+
Dictionary containing IMF metadata
|
|
12
|
+
|
|
13
|
+
!!! note "Compliant IMF metadata"
|
|
14
|
+
The current RDF provide IMF types as SHACL, but there are not any metadata describing
|
|
15
|
+
the actual content.
|
|
16
|
+
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
raw_metadata = {
|
|
20
|
+
"role": RoleTypes.information,
|
|
21
|
+
"schema": SchemaCompleteness.partial,
|
|
22
|
+
"prefix": "pca-imf",
|
|
23
|
+
"namespace": Namespace("http://posccaesar.org/imf/"),
|
|
24
|
+
"version": None,
|
|
25
|
+
"created": None,
|
|
26
|
+
"updated": None,
|
|
27
|
+
"title": "IMF - types",
|
|
28
|
+
"description": "IMF - types",
|
|
29
|
+
"creator": None,
|
|
30
|
+
"rights": None,
|
|
31
|
+
"license": None,
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
return make_metadata_compliant(raw_metadata)
|