cognite-neat 0.88.0__py3-none-any.whl → 0.88.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_version.py +1 -1
- cognite/neat/app/api/routers/configuration.py +1 -1
- cognite/neat/app/ui/neat-app/build/asset-manifest.json +7 -7
- cognite/neat/app/ui/neat-app/build/index.html +1 -1
- cognite/neat/app/ui/neat-app/build/static/css/{main.38a62222.css → main.72e3d92e.css} +2 -2
- cognite/neat/app/ui/neat-app/build/static/css/main.72e3d92e.css.map +1 -0
- cognite/neat/app/ui/neat-app/build/static/js/main.5a52cf09.js +3 -0
- cognite/neat/app/ui/neat-app/build/static/js/{main.ec7f72e2.js.LICENSE.txt → main.5a52cf09.js.LICENSE.txt} +0 -9
- cognite/neat/app/ui/neat-app/build/static/js/main.5a52cf09.js.map +1 -0
- cognite/neat/config.py +44 -27
- cognite/neat/exceptions.py +8 -2
- cognite/neat/graph/extractors/_classic_cdf/_assets.py +21 -73
- cognite/neat/graph/extractors/_classic_cdf/_base.py +102 -0
- cognite/neat/graph/extractors/_classic_cdf/_events.py +46 -42
- cognite/neat/graph/extractors/_classic_cdf/_files.py +41 -45
- cognite/neat/graph/extractors/_classic_cdf/_labels.py +75 -52
- cognite/neat/graph/extractors/_classic_cdf/_relationships.py +49 -27
- cognite/neat/graph/extractors/_classic_cdf/_sequences.py +47 -50
- cognite/neat/graph/extractors/_classic_cdf/_timeseries.py +47 -49
- cognite/neat/graph/loaders/_base.py +4 -4
- cognite/neat/graph/loaders/_rdf2asset.py +12 -14
- cognite/neat/graph/loaders/_rdf2dms.py +14 -10
- cognite/neat/graph/queries/_base.py +22 -29
- cognite/neat/graph/queries/_shared.py +1 -1
- cognite/neat/graph/stores/_base.py +19 -11
- cognite/neat/graph/transformers/_rdfpath.py +3 -2
- cognite/neat/issues/__init__.py +16 -0
- cognite/neat/{issues.py → issues/_base.py} +78 -2
- cognite/neat/issues/errors/external.py +21 -0
- cognite/neat/issues/errors/properties.py +75 -0
- cognite/neat/issues/errors/resources.py +123 -0
- cognite/neat/issues/errors/schema.py +0 -0
- cognite/neat/{rules/issues → issues}/formatters.py +9 -9
- cognite/neat/issues/neat_warnings/__init__.py +2 -0
- cognite/neat/issues/neat_warnings/identifier.py +27 -0
- cognite/neat/issues/neat_warnings/models.py +22 -0
- cognite/neat/issues/neat_warnings/properties.py +77 -0
- cognite/neat/issues/neat_warnings/resources.py +125 -0
- cognite/neat/rules/exporters/_rules2dms.py +3 -2
- cognite/neat/rules/exporters/_rules2ontology.py +28 -20
- cognite/neat/rules/exporters/_validation.py +15 -21
- cognite/neat/rules/importers/__init__.py +7 -3
- cognite/neat/rules/importers/_base.py +3 -3
- cognite/neat/rules/importers/_dms2rules.py +39 -18
- cognite/neat/rules/importers/_dtdl2rules/dtdl_converter.py +44 -53
- cognite/neat/rules/importers/_dtdl2rules/dtdl_importer.py +6 -5
- cognite/neat/rules/importers/_rdf/__init__.py +0 -0
- cognite/neat/rules/importers/_rdf/_imf2rules/__init__.py +3 -0
- cognite/neat/rules/importers/_rdf/_imf2rules/_imf2classes.py +82 -0
- cognite/neat/rules/importers/_rdf/_imf2rules/_imf2metadata.py +34 -0
- cognite/neat/rules/importers/_rdf/_imf2rules/_imf2properties.py +123 -0
- cognite/neat/rules/importers/{_owl2rules/_owl2rules.py → _rdf/_imf2rules/_imf2rules.py} +15 -11
- cognite/neat/rules/importers/{_inference2rules.py → _rdf/_inference2rules.py} +1 -1
- cognite/neat/rules/importers/_rdf/_owl2rules/_owl2classes.py +57 -0
- cognite/neat/rules/importers/_rdf/_owl2rules/_owl2metadata.py +68 -0
- cognite/neat/rules/importers/_rdf/_owl2rules/_owl2properties.py +59 -0
- cognite/neat/rules/importers/_rdf/_owl2rules/_owl2rules.py +76 -0
- cognite/neat/rules/importers/_rdf/_shared.py +586 -0
- cognite/neat/rules/importers/_spreadsheet2rules.py +31 -28
- cognite/neat/rules/importers/_yaml2rules.py +2 -1
- cognite/neat/rules/issues/__init__.py +1 -5
- cognite/neat/rules/issues/base.py +2 -21
- cognite/neat/rules/issues/dms.py +20 -134
- cognite/neat/rules/issues/ontology.py +298 -0
- cognite/neat/rules/issues/spreadsheet.py +51 -3
- cognite/neat/rules/issues/tables.py +72 -0
- cognite/neat/rules/models/_rdfpath.py +4 -4
- cognite/neat/rules/models/_types/_field.py +14 -21
- cognite/neat/rules/models/asset/_validation.py +1 -1
- cognite/neat/rules/models/dms/_schema.py +53 -30
- cognite/neat/rules/models/dms/_validation.py +2 -2
- cognite/neat/rules/models/entities.py +3 -0
- cognite/neat/rules/models/information/_rules.py +5 -4
- cognite/neat/rules/models/information/_validation.py +1 -1
- cognite/neat/utils/rdf_.py +17 -9
- cognite/neat/utils/regex_patterns.py +52 -0
- cognite/neat/workflows/steps/lib/current/rules_importer.py +73 -1
- cognite/neat/workflows/steps/lib/current/rules_validator.py +19 -7
- {cognite_neat-0.88.0.dist-info → cognite_neat-0.88.2.dist-info}/METADATA +2 -6
- {cognite_neat-0.88.0.dist-info → cognite_neat-0.88.2.dist-info}/RECORD +85 -72
- cognite/neat/app/ui/neat-app/build/static/css/main.38a62222.css.map +0 -1
- cognite/neat/app/ui/neat-app/build/static/js/main.ec7f72e2.js +0 -3
- cognite/neat/app/ui/neat-app/build/static/js/main.ec7f72e2.js.map +0 -1
- cognite/neat/graph/issues/loader.py +0 -104
- cognite/neat/graph/stores/_oxrdflib.py +0 -247
- cognite/neat/rules/exceptions.py +0 -2972
- cognite/neat/rules/importers/_owl2rules/_owl2classes.py +0 -215
- cognite/neat/rules/importers/_owl2rules/_owl2metadata.py +0 -213
- cognite/neat/rules/importers/_owl2rules/_owl2properties.py +0 -203
- cognite/neat/rules/issues/importing.py +0 -408
- cognite/neat/rules/models/_types/_base.py +0 -16
- cognite/neat/workflows/examples/Export_Rules_to_Ontology/workflow.yaml +0 -152
- cognite/neat/workflows/examples/Extract_DEXPI_Graph_and_Export_Rules/workflow.yaml +0 -139
- cognite/neat/workflows/examples/Ontology_to_Data_Model/workflow.yaml +0 -116
- /cognite/neat/{graph/issues → issues/errors}/__init__.py +0 -0
- /cognite/neat/rules/importers/{_owl2rules → _rdf/_owl2rules}/__init__.py +0 -0
- {cognite_neat-0.88.0.dist-info → cognite_neat-0.88.2.dist-info}/LICENSE +0 -0
- {cognite_neat-0.88.0.dist-info → cognite_neat-0.88.2.dist-info}/WHEEL +0 -0
- {cognite_neat-0.88.0.dist-info → cognite_neat-0.88.2.dist-info}/entry_points.txt +0 -0
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
|
|
3
|
+
from .base import NeatValidationError
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@dataclass(frozen=True)
|
|
7
|
+
class NotValidRDFPathError(NeatValidationError):
|
|
8
|
+
"""Provided `rdfpath` is not valid, i.e. it cannot be converted to SPARQL query.
|
|
9
|
+
|
|
10
|
+
Args:
|
|
11
|
+
rdf_path: `rdfpath` that raised exception
|
|
12
|
+
|
|
13
|
+
Notes:
|
|
14
|
+
Get familiar with `rdfpath` to avoid this exception.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
description = "Provided `rdfpath` is not valid, i.e. it cannot be converted to SPARQL query"
|
|
18
|
+
fix = "Get familiar with `rdfpath` and check if provided path is valid!"
|
|
19
|
+
rdf_path: str
|
|
20
|
+
|
|
21
|
+
def message(self) -> str:
|
|
22
|
+
message = f"{self.rdf_path} is not a valid rdfpath!"
|
|
23
|
+
|
|
24
|
+
message += f"\nDescription: {self.description}"
|
|
25
|
+
message += f"\nFix: {self.fix}"
|
|
26
|
+
return message
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass(frozen=True)
|
|
30
|
+
class NotValidTableLookUpError(NeatValidationError):
|
|
31
|
+
"""Provided `table lookup` is not valid, i.e. it cannot be converted to CDF lookup.
|
|
32
|
+
|
|
33
|
+
Args:
|
|
34
|
+
table_look_up: `table_look_up`, a part of `rawlookup`, that raised exception
|
|
35
|
+
|
|
36
|
+
Notes:
|
|
37
|
+
Get familiar with `rawlookup` and `rdfpath` to avoid this exception.
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
description = "Provided table lookup is not valid, i.e. it cannot be converted to CDF lookup"
|
|
41
|
+
fix = "Get familiar with RAW look up and RDF paths and check if provided rawlookup is valid"
|
|
42
|
+
table_look_up: str
|
|
43
|
+
|
|
44
|
+
def message(self) -> str:
|
|
45
|
+
message = f"{self.table_look_up} is not a valid table lookup"
|
|
46
|
+
|
|
47
|
+
message += f"\nDescription: {self.description}"
|
|
48
|
+
message += f"\nFix: {self.fix}"
|
|
49
|
+
return message
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@dataclass(frozen=True)
|
|
53
|
+
class NotValidRAWLookUpError(NeatValidationError):
|
|
54
|
+
"""Provided `rawlookup` is not valid, i.e. it cannot be converted to SPARQL query and CDF lookup
|
|
55
|
+
|
|
56
|
+
Args:
|
|
57
|
+
raw_look_up: `rawlookup` rule that raised exception
|
|
58
|
+
|
|
59
|
+
Notes:
|
|
60
|
+
Get familiar with `rawlookup` and `rdfpath` to avoid this exception.
|
|
61
|
+
"""
|
|
62
|
+
|
|
63
|
+
description = "Provided rawlookup is not valid, i.e. it cannot be converted to SPARQL query and CDF lookup"
|
|
64
|
+
fix = "Get familiar with `rawlookup` and `rdfpath` to avoid this exception"
|
|
65
|
+
raw_look_up: str
|
|
66
|
+
|
|
67
|
+
def message(self):
|
|
68
|
+
message = f"Invalid rawlookup expected traversal | table lookup, got {self.raw_look_up}"
|
|
69
|
+
|
|
70
|
+
message += f"\nDescription: {self.description}"
|
|
71
|
+
message += f"\nFix: {self.fix}"
|
|
72
|
+
return message
|
|
@@ -8,7 +8,7 @@ from typing import ClassVar, Literal
|
|
|
8
8
|
|
|
9
9
|
from pydantic import BaseModel, field_validator, model_serializer
|
|
10
10
|
|
|
11
|
-
from cognite.neat.rules import
|
|
11
|
+
from cognite.neat.rules.issues.tables import NotValidRAWLookUpError, NotValidRDFPathError, NotValidTableLookUpError
|
|
12
12
|
|
|
13
13
|
if sys.version_info >= (3, 11):
|
|
14
14
|
from enum import StrEnum
|
|
@@ -313,7 +313,7 @@ def parse_traversal(raw: str) -> SelfReferenceProperty | SingleProperty | Hop:
|
|
|
313
313
|
elif result := HOP_REGEX_COMPILED.match(raw):
|
|
314
314
|
return Hop.from_string(class_=result.group("origin"), traversal=result.group(_traversal))
|
|
315
315
|
else:
|
|
316
|
-
raise
|
|
316
|
+
raise NotValidRDFPathError(raw).as_pydantic_exception()
|
|
317
317
|
|
|
318
318
|
|
|
319
319
|
def parse_table_lookup(raw: str) -> TableLookup:
|
|
@@ -323,7 +323,7 @@ def parse_table_lookup(raw: str) -> TableLookup:
|
|
|
323
323
|
key=result.group(Lookup.key),
|
|
324
324
|
value=result.group(Lookup.value),
|
|
325
325
|
)
|
|
326
|
-
raise
|
|
326
|
+
raise NotValidTableLookUpError(raw).as_pydantic_exception()
|
|
327
327
|
|
|
328
328
|
|
|
329
329
|
def parse_rule(rule_raw: str, rule_type: TransformationRuleType | None) -> RDFPath:
|
|
@@ -334,7 +334,7 @@ def parse_rule(rule_raw: str, rule_type: TransformationRuleType | None) -> RDFPa
|
|
|
334
334
|
case TransformationRuleType.rawlookup:
|
|
335
335
|
rule_raw = rule_raw.replace(" ", "")
|
|
336
336
|
if Counter(rule_raw).get("|") != 1:
|
|
337
|
-
raise
|
|
337
|
+
raise NotValidRAWLookUpError(rule_raw).as_pydantic_exception()
|
|
338
338
|
traversal, table_lookup = rule_raw.split("|")
|
|
339
339
|
return RawLookup(
|
|
340
340
|
traversal=parse_traversal(traversal),
|
|
@@ -1,7 +1,6 @@
|
|
|
1
|
-
import re
|
|
2
1
|
import warnings
|
|
3
2
|
from collections.abc import Callable
|
|
4
|
-
from typing import Annotated, Any
|
|
3
|
+
from typing import Annotated, Any
|
|
5
4
|
|
|
6
5
|
import rdflib
|
|
7
6
|
from pydantic import (
|
|
@@ -17,11 +16,10 @@ from pydantic import (
|
|
|
17
16
|
from pydantic.functional_serializers import PlainSerializer
|
|
18
17
|
from pydantic_core import PydanticCustomError
|
|
19
18
|
|
|
20
|
-
from cognite.neat.
|
|
21
|
-
from cognite.neat.rules.issues.
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
MORE_THAN_ONE_NONE_ALPHANUMERIC_REGEX,
|
|
19
|
+
from cognite.neat.issues.neat_warnings.identifier import RegexViolationWarning
|
|
20
|
+
from cognite.neat.rules.issues.spreadsheet import RegexViolationError
|
|
21
|
+
from cognite.neat.utils.regex_patterns import (
|
|
22
|
+
PATTERNS,
|
|
25
23
|
PREFIX_COMPLIANCE_REGEX,
|
|
26
24
|
PROPERTY_ID_COMPLIANCE_REGEX,
|
|
27
25
|
VERSION_COMPLIANCE_REGEX,
|
|
@@ -74,11 +72,7 @@ NamespaceType = Annotated[
|
|
|
74
72
|
PrefixType = Annotated[
|
|
75
73
|
str,
|
|
76
74
|
StringConstraints(pattern=PREFIX_COMPLIANCE_REGEX),
|
|
77
|
-
_custom_error(
|
|
78
|
-
lambda _, value: exceptions.PrefixesRegexViolation(
|
|
79
|
-
cast(list[str], [value]), PREFIX_COMPLIANCE_REGEX
|
|
80
|
-
).to_pydantic_custom_error()
|
|
81
|
-
),
|
|
75
|
+
_custom_error(lambda _, value: RegexViolationError(value, PREFIX_COMPLIANCE_REGEX).as_pydantic_exception()),
|
|
82
76
|
]
|
|
83
77
|
|
|
84
78
|
ExternalIdType = Annotated[
|
|
@@ -89,19 +83,18 @@ ExternalIdType = Annotated[
|
|
|
89
83
|
VersionType = Annotated[
|
|
90
84
|
str,
|
|
91
85
|
StringConstraints(pattern=VERSION_COMPLIANCE_REGEX),
|
|
92
|
-
_custom_error(
|
|
93
|
-
lambda _, value: exceptions.VersionRegexViolation(
|
|
94
|
-
version=cast(str, value), regex_expression=VERSION_COMPLIANCE_REGEX
|
|
95
|
-
).to_pydantic_custom_error()
|
|
96
|
-
),
|
|
86
|
+
_custom_error(lambda _, value: RegexViolationError(value, VERSION_COMPLIANCE_REGEX).as_pydantic_exception()),
|
|
97
87
|
]
|
|
98
88
|
|
|
99
89
|
|
|
100
90
|
def _property_validation(value: str) -> str:
|
|
101
|
-
if not
|
|
102
|
-
_raise(
|
|
103
|
-
if
|
|
104
|
-
warnings.warn(
|
|
91
|
+
if not PATTERNS.property_id_compliance.match(value):
|
|
92
|
+
_raise(RegexViolationError(value, PROPERTY_ID_COMPLIANCE_REGEX).as_pydantic_exception())
|
|
93
|
+
if PATTERNS.more_than_one_alphanumeric.search(value):
|
|
94
|
+
warnings.warn(
|
|
95
|
+
RegexViolationWarning(value, PROPERTY_ID_COMPLIANCE_REGEX, "property", "MoreThanOneNonAlphanumeric"),
|
|
96
|
+
stacklevel=2,
|
|
97
|
+
)
|
|
105
98
|
return value
|
|
106
99
|
|
|
107
100
|
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
from graphlib import CycleError
|
|
2
2
|
from typing import cast
|
|
3
3
|
|
|
4
|
+
from cognite.neat.issues import IssueList
|
|
4
5
|
from cognite.neat.rules import issues
|
|
5
|
-
from cognite.neat.rules.issues.base import IssueList
|
|
6
6
|
from cognite.neat.rules.models._base import SheetList
|
|
7
7
|
from cognite.neat.rules.models.asset._rules import AssetProperty, AssetRules
|
|
8
8
|
from cognite.neat.rules.models.entities import AssetEntity, AssetFields, ClassEntity
|
|
@@ -25,20 +25,16 @@ from cognite.client.data_classes.data_modeling.views import (
|
|
|
25
25
|
)
|
|
26
26
|
from cognite.client.data_classes.transformations.common import Edges, EdgeType, Nodes, ViewInfo
|
|
27
27
|
|
|
28
|
+
from cognite.neat.issues import NeatError
|
|
29
|
+
from cognite.neat.issues.errors.properties import ReferredPropertyNotFoundError
|
|
30
|
+
from cognite.neat.issues.errors.resources import ReferredResourceNotFoundError
|
|
31
|
+
from cognite.neat.issues.neat_warnings.resources import FailedLoadingResourcesWarning, MultipleResourcesWarning
|
|
28
32
|
from cognite.neat.rules import issues
|
|
29
33
|
from cognite.neat.rules.issues.dms import (
|
|
30
34
|
ContainerPropertyUsedMultipleTimesError,
|
|
31
35
|
DirectRelationMissingSourceWarning,
|
|
32
|
-
DMSSchemaError,
|
|
33
36
|
DuplicatedViewInDataModelError,
|
|
34
37
|
IncompleteSchemaError,
|
|
35
|
-
MissingContainerError,
|
|
36
|
-
MissingContainerPropertyError,
|
|
37
|
-
MissingEdgeViewError,
|
|
38
|
-
MissingParentViewError,
|
|
39
|
-
MissingSourceViewError,
|
|
40
|
-
MissingSpaceError,
|
|
41
|
-
MissingViewError,
|
|
42
38
|
MissingViewInModelWarning,
|
|
43
39
|
)
|
|
44
40
|
from cognite.neat.rules.models.data_types import _DATA_TYPE_BY_DMS_TYPE
|
|
@@ -168,14 +164,10 @@ class DMSSchema:
|
|
|
168
164
|
connection_referenced_view_ids |= cls._connection_references(view)
|
|
169
165
|
connection_referenced_view_ids = connection_referenced_view_ids - existing_view_ids
|
|
170
166
|
if connection_referenced_view_ids:
|
|
171
|
-
warnings.warn(
|
|
172
|
-
MissingViewInModelWarning(data_model.as_id(), connection_referenced_view_ids), UserWarning, stacklevel=2
|
|
173
|
-
)
|
|
167
|
+
warnings.warn(MissingViewInModelWarning(data_model.as_id(), connection_referenced_view_ids), stacklevel=2)
|
|
174
168
|
connection_referenced_views = view_loader.retrieve(list(connection_referenced_view_ids))
|
|
175
169
|
if failed := connection_referenced_view_ids - set(connection_referenced_views.as_ids()):
|
|
176
|
-
warnings.warn(
|
|
177
|
-
issues.importing.FailedImportWarning({repr(v) for v in failed}), UserWarning, stacklevel=2
|
|
178
|
-
)
|
|
170
|
+
warnings.warn(FailedLoadingResourcesWarning[dm.ViewId](frozenset(failed), "View"), stacklevel=2)
|
|
179
171
|
views.extend(connection_referenced_views)
|
|
180
172
|
|
|
181
173
|
# We need to include parent views in the schema to make sure that the schema is valid.
|
|
@@ -432,8 +424,9 @@ class DMSSchema:
|
|
|
432
424
|
if attr.name == "data_model":
|
|
433
425
|
if isinstance(items, list) and len(items) > 1:
|
|
434
426
|
warnings.warn(
|
|
435
|
-
|
|
436
|
-
[item.get("externalId", "Unknown") for item in items]
|
|
427
|
+
MultipleResourcesWarning[str](
|
|
428
|
+
frozenset([item.get("externalId", "Unknown") for item in items]),
|
|
429
|
+
"DataModel",
|
|
437
430
|
),
|
|
438
431
|
stacklevel=2,
|
|
439
432
|
)
|
|
@@ -522,8 +515,8 @@ class DMSSchema:
|
|
|
522
515
|
else:
|
|
523
516
|
raise ValueError(f"Cannot sort item of type {type(item)}")
|
|
524
517
|
|
|
525
|
-
def validate(self) -> list[
|
|
526
|
-
errors: set[
|
|
518
|
+
def validate(self) -> list[NeatError]:
|
|
519
|
+
errors: set[NeatError] = set()
|
|
527
520
|
defined_spaces = self.spaces.copy()
|
|
528
521
|
defined_containers = self.containers.copy()
|
|
529
522
|
defined_views = self.views.copy()
|
|
@@ -535,28 +528,42 @@ class DMSSchema:
|
|
|
535
528
|
|
|
536
529
|
for container in self.containers.values():
|
|
537
530
|
if container.space not in defined_spaces:
|
|
538
|
-
errors.add(
|
|
531
|
+
errors.add(
|
|
532
|
+
ReferredResourceNotFoundError[str, dm.ContainerId](
|
|
533
|
+
container.space, "Space", container.as_id(), "Container"
|
|
534
|
+
)
|
|
535
|
+
)
|
|
539
536
|
|
|
540
537
|
for view in self.views.values():
|
|
541
538
|
view_id = view.as_id()
|
|
542
539
|
if view.space not in defined_spaces:
|
|
543
|
-
errors.add(
|
|
540
|
+
errors.add(ReferredResourceNotFoundError[str, dm.ViewId](view.space, "Space", view_id, "View"))
|
|
544
541
|
|
|
545
542
|
for parent in view.implements or []:
|
|
546
543
|
if parent not in defined_views:
|
|
547
|
-
errors.add(
|
|
544
|
+
errors.add(
|
|
545
|
+
ReferredPropertyNotFoundError[dm.ViewId, dm.ViewId](
|
|
546
|
+
parent, "View", view_id, "View", property_name="implements"
|
|
547
|
+
)
|
|
548
|
+
)
|
|
548
549
|
|
|
549
550
|
for prop_name, prop in (view.properties or {}).items():
|
|
550
551
|
if isinstance(prop, dm.MappedPropertyApply):
|
|
551
552
|
ref_container = defined_containers.get(prop.container)
|
|
552
553
|
if ref_container is None:
|
|
553
|
-
errors.add(
|
|
554
|
+
errors.add(
|
|
555
|
+
ReferredResourceNotFoundError[dm.ContainerId, dm.ViewId](
|
|
556
|
+
prop.container, "Container", view_id, "View"
|
|
557
|
+
)
|
|
558
|
+
)
|
|
554
559
|
elif prop.container_property_identifier not in ref_container.properties:
|
|
555
560
|
errors.add(
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
561
|
+
ReferredPropertyNotFoundError[dm.ContainerId, dm.ViewId](
|
|
562
|
+
prop.container,
|
|
563
|
+
"Container",
|
|
564
|
+
view_id,
|
|
565
|
+
"View",
|
|
566
|
+
property_name=prop.container_property_identifier,
|
|
560
567
|
)
|
|
561
568
|
)
|
|
562
569
|
else:
|
|
@@ -568,14 +575,22 @@ class DMSSchema:
|
|
|
568
575
|
)
|
|
569
576
|
|
|
570
577
|
if isinstance(prop, dm.EdgeConnectionApply) and prop.source not in defined_views:
|
|
571
|
-
errors.add(
|
|
578
|
+
errors.add(
|
|
579
|
+
ReferredPropertyNotFoundError[dm.ViewId, dm.ViewId](
|
|
580
|
+
prop.source, "View", view_id, "View", property_name=prop_name
|
|
581
|
+
)
|
|
582
|
+
)
|
|
572
583
|
|
|
573
584
|
if (
|
|
574
585
|
isinstance(prop, dm.EdgeConnectionApply)
|
|
575
586
|
and prop.edge_source is not None
|
|
576
587
|
and prop.edge_source not in defined_views
|
|
577
588
|
):
|
|
578
|
-
errors.add(
|
|
589
|
+
errors.add(
|
|
590
|
+
ReferredPropertyNotFoundError[dm.ViewId, dm.ViewId](
|
|
591
|
+
prop.edge_source, "View", view_id, "View", property_name=prop_name
|
|
592
|
+
)
|
|
593
|
+
)
|
|
579
594
|
|
|
580
595
|
# This allows for multiple view properties to be mapped to the same container property,
|
|
581
596
|
# as long as they have different external_id, otherwise this will lead to raising
|
|
@@ -610,13 +625,21 @@ class DMSSchema:
|
|
|
610
625
|
if self.data_model:
|
|
611
626
|
model = self.data_model
|
|
612
627
|
if model.space not in defined_spaces:
|
|
613
|
-
errors.add(
|
|
628
|
+
errors.add(
|
|
629
|
+
ReferredResourceNotFoundError[str, dm.DataModelId](
|
|
630
|
+
model.space, "Space", model.as_id(), "Data Model"
|
|
631
|
+
)
|
|
632
|
+
)
|
|
614
633
|
|
|
615
634
|
view_counts: dict[dm.ViewId, int] = defaultdict(int)
|
|
616
635
|
for view_id_or_class in model.views or []:
|
|
617
636
|
view_id = view_id_or_class if isinstance(view_id_or_class, dm.ViewId) else view_id_or_class.as_id()
|
|
618
637
|
if view_id not in defined_views:
|
|
619
|
-
errors.add(
|
|
638
|
+
errors.add(
|
|
639
|
+
ReferredResourceNotFoundError[dm.ViewId, dm.DataModelId](
|
|
640
|
+
view_id, "View", model.as_id(), "DataModel"
|
|
641
|
+
)
|
|
642
|
+
)
|
|
620
643
|
view_counts[view_id] += 1
|
|
621
644
|
|
|
622
645
|
for view_id, count in view_counts.items():
|
|
@@ -3,8 +3,8 @@ from typing import Any, ClassVar
|
|
|
3
3
|
|
|
4
4
|
from cognite.client import data_modeling as dm
|
|
5
5
|
|
|
6
|
+
from cognite.neat.issues import IssueList, NeatIssueList
|
|
6
7
|
from cognite.neat.rules import issues
|
|
7
|
-
from cognite.neat.rules.issues import IssueList
|
|
8
8
|
from cognite.neat.rules.models._base import DataModelType, ExtensionCategory, SchemaCompleteness
|
|
9
9
|
from cognite.neat.rules.models._constants import DMS_CONTAINER_SIZE_LIMIT
|
|
10
10
|
from cognite.neat.rules.models.data_types import DataType
|
|
@@ -31,7 +31,7 @@ class DMSPostValidation:
|
|
|
31
31
|
self.views = rules.views
|
|
32
32
|
self.issue_list = IssueList()
|
|
33
33
|
|
|
34
|
-
def validate(self) ->
|
|
34
|
+
def validate(self) -> NeatIssueList:
|
|
35
35
|
self._validate_raw_filter()
|
|
36
36
|
self._consistent_container_properties()
|
|
37
37
|
|
|
@@ -418,6 +418,9 @@ class ViewEntity(DMSVersionedEntity[ViewId]):
|
|
|
418
418
|
) -> ViewId:
|
|
419
419
|
return ViewId(space=self.space, external_id=self.external_id, version=self.version)
|
|
420
420
|
|
|
421
|
+
def to_property_id(self, property_id: str) -> PropertyId:
|
|
422
|
+
return PropertyId(source=self.as_id(), property=property_id)
|
|
423
|
+
|
|
421
424
|
@classmethod
|
|
422
425
|
def from_id(cls, id: ViewId, default_version: str | None = None) -> "ViewEntity":
|
|
423
426
|
if id.version is not None:
|
|
@@ -9,7 +9,8 @@ from rdflib import Namespace
|
|
|
9
9
|
|
|
10
10
|
from cognite.neat.constants import get_default_prefixes
|
|
11
11
|
from cognite.neat.issues import MultiValueError
|
|
12
|
-
from cognite.neat.rules import
|
|
12
|
+
from cognite.neat.rules import issues
|
|
13
|
+
from cognite.neat.rules.issues.spreadsheet import DefaultValueTypeNotProperError
|
|
13
14
|
from cognite.neat.rules.models._base import (
|
|
14
15
|
BaseMetadata,
|
|
15
16
|
BaseRules,
|
|
@@ -227,11 +228,11 @@ class InformationProperty(SheetEntity):
|
|
|
227
228
|
self.default = self.value_type.python(self.default)
|
|
228
229
|
|
|
229
230
|
except Exception:
|
|
230
|
-
|
|
231
|
+
raise DefaultValueTypeNotProperError(
|
|
231
232
|
self.property_,
|
|
232
233
|
type(self.default),
|
|
233
|
-
self.value_type.python,
|
|
234
|
-
)
|
|
234
|
+
str(self.value_type.python),
|
|
235
|
+
).as_exception() from None
|
|
235
236
|
return self
|
|
236
237
|
|
|
237
238
|
@property
|
|
@@ -2,8 +2,8 @@ import itertools
|
|
|
2
2
|
from collections import Counter
|
|
3
3
|
from typing import cast
|
|
4
4
|
|
|
5
|
+
from cognite.neat.issues import IssueList
|
|
5
6
|
from cognite.neat.rules import issues
|
|
6
|
-
from cognite.neat.rules.issues import IssueList
|
|
7
7
|
from cognite.neat.rules.models._base import DataModelType, SchemaCompleteness
|
|
8
8
|
from cognite.neat.rules.models.entities import ClassEntity, EntityTypes, UnknownEntity
|
|
9
9
|
from cognite.neat.utils.rdf_ import get_inheritance_path
|
cognite/neat/utils/rdf_.py
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import re
|
|
2
|
-
from
|
|
2
|
+
from collections.abc import Iterable
|
|
3
|
+
from typing import Any, Literal, TypeAlias, overload
|
|
3
4
|
|
|
5
|
+
from cognite.client.utils.useful_types import SequenceNotStr
|
|
4
6
|
from pydantic import HttpUrl, TypeAdapter, ValidationError
|
|
5
7
|
from rdflib import Literal as RdfLiteral
|
|
6
8
|
from rdflib import Namespace, URIRef
|
|
@@ -10,7 +12,8 @@ Triple: TypeAlias = tuple[URIRef, URIRef, RdfLiteral | URIRef]
|
|
|
10
12
|
|
|
11
13
|
@overload
|
|
12
14
|
def remove_namespace_from_uri(
|
|
13
|
-
|
|
15
|
+
URI: URIRef | str,
|
|
16
|
+
*,
|
|
14
17
|
special_separator: str = "#_",
|
|
15
18
|
validation: Literal["full", "prefix"] = "prefix",
|
|
16
19
|
) -> str: ...
|
|
@@ -18,17 +21,19 @@ def remove_namespace_from_uri(
|
|
|
18
21
|
|
|
19
22
|
@overload
|
|
20
23
|
def remove_namespace_from_uri(
|
|
21
|
-
|
|
24
|
+
URI: SequenceNotStr[URIRef | str],
|
|
25
|
+
*,
|
|
22
26
|
special_separator: str = "#_",
|
|
23
27
|
validation: Literal["full", "prefix"] = "prefix",
|
|
24
|
-
) ->
|
|
28
|
+
) -> list[str]: ...
|
|
25
29
|
|
|
26
30
|
|
|
27
31
|
def remove_namespace_from_uri(
|
|
28
|
-
|
|
32
|
+
URI: URIRef | str | SequenceNotStr[URIRef | str],
|
|
33
|
+
*,
|
|
29
34
|
special_separator: str = "#_",
|
|
30
35
|
validation: Literal["full", "prefix"] = "prefix",
|
|
31
|
-
) ->
|
|
36
|
+
) -> str | list[str]:
|
|
32
37
|
"""Removes namespace from URI
|
|
33
38
|
|
|
34
39
|
Args
|
|
@@ -51,11 +56,14 @@ def remove_namespace_from_uri(
|
|
|
51
56
|
>>> remove_namespace_from_uri("http://www.example.org/index.html#section2", "http://www.example.org/index.html#section3")
|
|
52
57
|
('section2', 'section3')
|
|
53
58
|
"""
|
|
59
|
+
is_single = False
|
|
60
|
+
uris: Iterable[str | URIRef]
|
|
54
61
|
if isinstance(URI, str | URIRef):
|
|
55
62
|
uris = (URI,)
|
|
56
|
-
|
|
63
|
+
is_single = True
|
|
64
|
+
elif isinstance(URI, SequenceNotStr):
|
|
57
65
|
# Assume that all elements in the tuple are of the same type following type hint
|
|
58
|
-
uris =
|
|
66
|
+
uris = URI
|
|
59
67
|
else:
|
|
60
68
|
raise TypeError(f"URI must be of type URIRef or str, got {type(URI)}")
|
|
61
69
|
|
|
@@ -73,7 +81,7 @@ def remove_namespace_from_uri(
|
|
|
73
81
|
else:
|
|
74
82
|
output.append(str(u))
|
|
75
83
|
|
|
76
|
-
return
|
|
84
|
+
return output[0] if is_single else output
|
|
77
85
|
|
|
78
86
|
|
|
79
87
|
def get_namespace(URI: URIRef, special_separator: str = "#_") -> str:
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import re
|
|
2
|
+
from functools import cached_property
|
|
3
|
+
|
|
4
|
+
MORE_THAN_ONE_NONE_ALPHANUMERIC_REGEX = r"([_-]{2,})"
|
|
5
|
+
PREFIX_COMPLIANCE_REGEX = r"^([a-zA-Z]+)([a-zA-Z0-9]*[_-]{0,1}[a-zA-Z0-9_-]*)([a-zA-Z0-9]*)$"
|
|
6
|
+
|
|
7
|
+
VIEW_ID_COMPLIANCE_REGEX = (
|
|
8
|
+
r"(?!^(Query|Mutation|Subscription|String|Int32|Int64|Int|Float32|Float64|Float|"
|
|
9
|
+
r"Timestamp|JSONObject|Date|Numeric|Boolean|PageInfo|File|Sequence|TimeSeries)$)"
|
|
10
|
+
r"(^[a-zA-Z][a-zA-Z0-9_]{0,253}[a-zA-Z0-9]?$)"
|
|
11
|
+
)
|
|
12
|
+
DMS_PROPERTY_ID_COMPLIANCE_REGEX = (
|
|
13
|
+
r"(?!^(space|externalId|createdTime|lastUpdatedTime|deletedTime|edge_id|"
|
|
14
|
+
r"node_id|project_id|property_group|seq|tg_table_name|extensions)$)"
|
|
15
|
+
r"(^[a-zA-Z][a-zA-Z0-9_]{0,253}[a-zA-Z0-9]?$)"
|
|
16
|
+
)
|
|
17
|
+
CLASS_ID_COMPLIANCE_REGEX = r"(?!^(Class|class)$)(^[a-zA-Z][a-zA-Z0-9._-]{0,253}[a-zA-Z0-9]?$)"
|
|
18
|
+
PROPERTY_ID_COMPLIANCE_REGEX = r"^(\*)|(?!^(Property|property)$)(^[a-zA-Z][a-zA-Z0-9._-]{0,253}[a-zA-Z0-9]?$)"
|
|
19
|
+
VERSION_COMPLIANCE_REGEX = r"^[a-zA-Z0-9]([.a-zA-Z0-9_-]{0,41}[a-zA-Z0-9])?$"
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class _Patterns:
|
|
23
|
+
@cached_property
|
|
24
|
+
def more_than_one_alphanumeric(self) -> re.Pattern:
|
|
25
|
+
return re.compile(MORE_THAN_ONE_NONE_ALPHANUMERIC_REGEX)
|
|
26
|
+
|
|
27
|
+
@cached_property
|
|
28
|
+
def prefix_compliance(self) -> re.Pattern[str]:
|
|
29
|
+
return re.compile(PREFIX_COMPLIANCE_REGEX)
|
|
30
|
+
|
|
31
|
+
@cached_property
|
|
32
|
+
def view_id_compliance(self) -> re.Pattern[str]:
|
|
33
|
+
return re.compile(VIEW_ID_COMPLIANCE_REGEX)
|
|
34
|
+
|
|
35
|
+
@cached_property
|
|
36
|
+
def dms_property_id_compliance(self) -> re.Pattern[str]:
|
|
37
|
+
return re.compile(DMS_PROPERTY_ID_COMPLIANCE_REGEX)
|
|
38
|
+
|
|
39
|
+
@cached_property
|
|
40
|
+
def class_id_compliance(self) -> re.Pattern[str]:
|
|
41
|
+
return re.compile(CLASS_ID_COMPLIANCE_REGEX)
|
|
42
|
+
|
|
43
|
+
@cached_property
|
|
44
|
+
def property_id_compliance(self) -> re.Pattern[str]:
|
|
45
|
+
return re.compile(PROPERTY_ID_COMPLIANCE_REGEX)
|
|
46
|
+
|
|
47
|
+
@cached_property
|
|
48
|
+
def version_compliance(self) -> re.Pattern[str]:
|
|
49
|
+
return re.compile(VERSION_COMPLIANCE_REGEX)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
PATTERNS = _Patterns()
|
|
@@ -5,8 +5,8 @@ from typing import ClassVar
|
|
|
5
5
|
from cognite.client import CogniteClient
|
|
6
6
|
from cognite.client.data_classes.data_modeling import DataModelId
|
|
7
7
|
|
|
8
|
+
from cognite.neat.issues.formatters import FORMATTER_BY_NAME
|
|
8
9
|
from cognite.neat.rules import importers
|
|
9
|
-
from cognite.neat.rules.issues.formatters import FORMATTER_BY_NAME
|
|
10
10
|
from cognite.neat.rules.models import RoleTypes
|
|
11
11
|
from cognite.neat.rules.models.entities import DataModelEntity, DMSUnknownEntity
|
|
12
12
|
from cognite.neat.workflows._exceptions import StepNotInitialized
|
|
@@ -19,6 +19,7 @@ CATEGORY = __name__.split(".")[-1].replace("_", " ").title()
|
|
|
19
19
|
__all__ = [
|
|
20
20
|
"ExcelToRules",
|
|
21
21
|
"OntologyToRules",
|
|
22
|
+
"IMFToRules",
|
|
22
23
|
"DMSToRules",
|
|
23
24
|
"RulesInferenceFromRdfFile",
|
|
24
25
|
]
|
|
@@ -162,6 +163,77 @@ class OntologyToRules(Step):
|
|
|
162
163
|
return FlowMessage(output_text=output_text), MultiRuleData.from_rules(rules)
|
|
163
164
|
|
|
164
165
|
|
|
166
|
+
class IMFToRules(Step):
|
|
167
|
+
"""This step import rules from the IMF-types and validates them."""
|
|
168
|
+
|
|
169
|
+
description = "This step imports rules from an RDF file "
|
|
170
|
+
version = "private-beta"
|
|
171
|
+
category = CATEGORY
|
|
172
|
+
configurables: ClassVar[list[Configurable]] = [
|
|
173
|
+
Configurable(
|
|
174
|
+
name="File name",
|
|
175
|
+
value="",
|
|
176
|
+
label="""Full file name of the RDF-file containing the IMF-types in the rules folder.
|
|
177
|
+
If not provided, step will attempt to get file name from payload
|
|
178
|
+
of 'File Uploader' step (if exist)""",
|
|
179
|
+
),
|
|
180
|
+
Configurable(
|
|
181
|
+
name="Report formatter",
|
|
182
|
+
value=next(iter(FORMATTER_BY_NAME.keys())),
|
|
183
|
+
label="The format of the report for the validation of the rules",
|
|
184
|
+
options=list(FORMATTER_BY_NAME),
|
|
185
|
+
),
|
|
186
|
+
Configurable(
|
|
187
|
+
name="Role",
|
|
188
|
+
value="infer",
|
|
189
|
+
label="For what role Rules are intended?",
|
|
190
|
+
options=["infer", *RoleTypes.__members__.keys()],
|
|
191
|
+
),
|
|
192
|
+
]
|
|
193
|
+
|
|
194
|
+
def run(self, flow_message: FlowMessage) -> (FlowMessage, MultiRuleData): # type: ignore[syntax, override]
|
|
195
|
+
if self.configs is None or self.data_store_path is None:
|
|
196
|
+
raise StepNotInitialized(type(self).__name__)
|
|
197
|
+
|
|
198
|
+
file_name = self.configs.get("File name", None)
|
|
199
|
+
|
|
200
|
+
full_path = flow_message.payload.get("full_path", None) if flow_message.payload else None
|
|
201
|
+
|
|
202
|
+
if file_name:
|
|
203
|
+
rules_file_path = self.config.rules_store_path / file_name
|
|
204
|
+
|
|
205
|
+
elif full_path:
|
|
206
|
+
rules_file_path = full_path
|
|
207
|
+
else:
|
|
208
|
+
error_text = "Expected either 'File name' in the step config or 'File uploader' step uploading Excel Rules."
|
|
209
|
+
return FlowMessage(error_text=error_text, step_execution_status=StepExecutionStatus.ABORT_AND_FAIL)
|
|
210
|
+
|
|
211
|
+
# if role is None, it will be inferred from the rules file
|
|
212
|
+
role = self.configs.get("Role")
|
|
213
|
+
role_enum = None
|
|
214
|
+
if role != "infer" and role is not None:
|
|
215
|
+
role_enum = RoleTypes[role]
|
|
216
|
+
|
|
217
|
+
ontology_importer = importers.IMFImporter(filepath=rules_file_path)
|
|
218
|
+
rules, issues = ontology_importer.to_rules(errors="continue", role=role_enum)
|
|
219
|
+
|
|
220
|
+
if rules is None:
|
|
221
|
+
output_dir = self.config.staging_path
|
|
222
|
+
report_writer = FORMATTER_BY_NAME[self.configs["Report formatter"]]()
|
|
223
|
+
report_writer.write_to_file(issues, file_or_dir_path=output_dir)
|
|
224
|
+
report_file = report_writer.default_file_name
|
|
225
|
+
error_text = (
|
|
226
|
+
"<p></p>"
|
|
227
|
+
f'<a href="/data/staging/{report_file}?{time.time()}" '
|
|
228
|
+
f'target="_blank">Failed to validate rules, click here for report</a>'
|
|
229
|
+
)
|
|
230
|
+
return FlowMessage(error_text=error_text, step_execution_status=StepExecutionStatus.ABORT_AND_FAIL)
|
|
231
|
+
|
|
232
|
+
output_text = "Rules validation passed successfully!"
|
|
233
|
+
|
|
234
|
+
return FlowMessage(output_text=output_text), MultiRuleData.from_rules(rules)
|
|
235
|
+
|
|
236
|
+
|
|
165
237
|
class DMSToRules(Step):
|
|
166
238
|
"""This step imports rules from CDF Data Model"""
|
|
167
239
|
|