cognite-neat 0.88.1__py3-none-any.whl → 0.88.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_version.py +1 -1
- cognite/neat/exceptions.py +2 -2
- cognite/neat/graph/loaders/_base.py +4 -4
- cognite/neat/graph/loaders/_rdf2asset.py +12 -14
- cognite/neat/graph/loaders/_rdf2dms.py +14 -10
- cognite/neat/issues/__init__.py +16 -0
- cognite/neat/{issues.py → issues/_base.py} +73 -5
- cognite/neat/issues/errors/external.py +21 -0
- cognite/neat/issues/errors/properties.py +75 -0
- cognite/neat/issues/errors/resources.py +123 -0
- cognite/neat/issues/errors/schema.py +0 -0
- cognite/neat/{rules/issues → issues}/formatters.py +9 -9
- cognite/neat/issues/neat_warnings/__init__.py +2 -0
- cognite/neat/issues/neat_warnings/identifier.py +27 -0
- cognite/neat/issues/neat_warnings/models.py +22 -0
- cognite/neat/issues/neat_warnings/properties.py +77 -0
- cognite/neat/issues/neat_warnings/resources.py +125 -0
- cognite/neat/rules/exporters/_rules2dms.py +3 -2
- cognite/neat/rules/exporters/_validation.py +2 -2
- cognite/neat/rules/importers/__init__.py +7 -3
- cognite/neat/rules/importers/_base.py +3 -3
- cognite/neat/rules/importers/_dms2rules.py +39 -18
- cognite/neat/rules/importers/_dtdl2rules/dtdl_converter.py +44 -53
- cognite/neat/rules/importers/_dtdl2rules/dtdl_importer.py +6 -5
- cognite/neat/rules/importers/_rdf/__init__.py +0 -0
- cognite/neat/rules/importers/_rdf/_imf2rules/__init__.py +3 -0
- cognite/neat/rules/importers/_rdf/_imf2rules/_imf2classes.py +82 -0
- cognite/neat/rules/importers/_rdf/_imf2rules/_imf2metadata.py +34 -0
- cognite/neat/rules/importers/_rdf/_imf2rules/_imf2properties.py +123 -0
- cognite/neat/rules/importers/{_owl2rules/_owl2rules.py → _rdf/_imf2rules/_imf2rules.py} +15 -11
- cognite/neat/rules/importers/{_inference2rules.py → _rdf/_inference2rules.py} +1 -1
- cognite/neat/rules/importers/_rdf/_owl2rules/_owl2classes.py +57 -0
- cognite/neat/rules/importers/_rdf/_owl2rules/_owl2metadata.py +68 -0
- cognite/neat/rules/importers/_rdf/_owl2rules/_owl2properties.py +59 -0
- cognite/neat/rules/importers/_rdf/_owl2rules/_owl2rules.py +76 -0
- cognite/neat/rules/importers/_rdf/_shared.py +586 -0
- cognite/neat/rules/importers/_spreadsheet2rules.py +1 -1
- cognite/neat/rules/importers/_yaml2rules.py +2 -1
- cognite/neat/rules/issues/__init__.py +1 -5
- cognite/neat/rules/issues/base.py +2 -21
- cognite/neat/rules/issues/dms.py +0 -134
- cognite/neat/rules/issues/spreadsheet.py +3 -3
- cognite/neat/rules/models/_types/_field.py +5 -2
- cognite/neat/rules/models/asset/_validation.py +1 -1
- cognite/neat/rules/models/dms/_schema.py +53 -30
- cognite/neat/rules/models/dms/_validation.py +2 -2
- cognite/neat/rules/models/entities.py +3 -0
- cognite/neat/rules/models/information/_validation.py +1 -1
- cognite/neat/workflows/steps/lib/current/rules_importer.py +73 -1
- cognite/neat/workflows/steps/lib/current/rules_validator.py +19 -7
- {cognite_neat-0.88.1.dist-info → cognite_neat-0.88.2.dist-info}/METADATA +1 -1
- {cognite_neat-0.88.1.dist-info → cognite_neat-0.88.2.dist-info}/RECORD +57 -42
- cognite/neat/graph/issues/loader.py +0 -104
- cognite/neat/rules/importers/_owl2rules/_owl2classes.py +0 -215
- cognite/neat/rules/importers/_owl2rules/_owl2metadata.py +0 -209
- cognite/neat/rules/importers/_owl2rules/_owl2properties.py +0 -203
- cognite/neat/rules/issues/importing.py +0 -423
- /cognite/neat/{graph/issues → issues/errors}/__init__.py +0 -0
- /cognite/neat/rules/importers/{_owl2rules → _rdf/_owl2rules}/__init__.py +0 -0
- {cognite_neat-0.88.1.dist-info → cognite_neat-0.88.2.dist-info}/LICENSE +0 -0
- {cognite_neat-0.88.1.dist-info → cognite_neat-0.88.2.dist-info}/WHEEL +0 -0
- {cognite_neat-0.88.1.dist-info → cognite_neat-0.88.2.dist-info}/entry_points.txt +0 -0
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
from typing import Any, Generic
|
|
3
|
+
|
|
4
|
+
from .resources import ResourceWarning, T_Identifier, T_ReferenceIdentifier
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@dataclass(frozen=True)
|
|
8
|
+
class PropertyTypeNotSupportedWarning(ResourceWarning[T_Identifier]):
|
|
9
|
+
"""The {resource_type} with identifier {identifier} has a property {property_name}
|
|
10
|
+
of unsupported type {property_type}. This will be ignored."""
|
|
11
|
+
|
|
12
|
+
property_name: str
|
|
13
|
+
property_type: str
|
|
14
|
+
|
|
15
|
+
def message(self) -> str:
|
|
16
|
+
return (self.__doc__ or "").format(
|
|
17
|
+
resource_type=self.resource_type,
|
|
18
|
+
identifier=repr(self.identifier),
|
|
19
|
+
property_name=self.property_name,
|
|
20
|
+
property_type=self.property_type,
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
def dump(self) -> dict[str, Any]:
|
|
24
|
+
output = super().dump()
|
|
25
|
+
output["property_name"] = self.property_name
|
|
26
|
+
output["property_type"] = self.property_type
|
|
27
|
+
return output
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass(frozen=True)
|
|
31
|
+
class ReferredPropertyNotFoundWarning(ResourceWarning, Generic[T_Identifier, T_ReferenceIdentifier]):
|
|
32
|
+
"""The {resource_type} with identifier {identifier} does not have a property {property_name} referred
|
|
33
|
+
to by {referred_type} {referred_by} does not exist. This will be ignored.
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
fix = "Ensure the {resource_type} {identifier} has a {property_name} property"
|
|
37
|
+
|
|
38
|
+
referred_by: T_ReferenceIdentifier
|
|
39
|
+
referred_type: str
|
|
40
|
+
property_name: str
|
|
41
|
+
|
|
42
|
+
def message(self) -> str:
|
|
43
|
+
return (self.__doc__ or "").format(
|
|
44
|
+
resource_type=self.resource_type,
|
|
45
|
+
identifier=repr(self.identifier),
|
|
46
|
+
referred_type=self.referred_type,
|
|
47
|
+
referred_by=repr(self.referred_by),
|
|
48
|
+
property_name=self.property_name,
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
def dump(self) -> dict[str, Any]:
|
|
52
|
+
output = super().dump()
|
|
53
|
+
output["resource_type"] = self.resource_type
|
|
54
|
+
output["identifier"] = self.identifier
|
|
55
|
+
output["referred_by"] = self.referred_by
|
|
56
|
+
output["referred_type"] = self.referred_type
|
|
57
|
+
output["property_name"] = self.property_name
|
|
58
|
+
return output
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
@dataclass(frozen=True)
|
|
62
|
+
class PropertyRedefinedWarning(ResourceWarning[T_Identifier]):
|
|
63
|
+
"""The {resource_type} with identifier {identifier} has a property {property_name} redefined."""
|
|
64
|
+
|
|
65
|
+
property_id: str
|
|
66
|
+
|
|
67
|
+
def message(self) -> str:
|
|
68
|
+
return (self.__doc__ or "").format(
|
|
69
|
+
resource_type=self.resource_type, identifier=repr(self.identifier), property_name=self.property_id
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
def dump(self) -> dict[str, Any]:
|
|
73
|
+
output = super().dump()
|
|
74
|
+
output["property_id"] = self.property_id
|
|
75
|
+
output["resource_type"] = self.resource_type
|
|
76
|
+
output["identifier"] = self.identifier
|
|
77
|
+
return output
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
from collections.abc import Hashable
|
|
2
|
+
from dataclasses import dataclass
|
|
3
|
+
from typing import Any, Generic, TypeVar
|
|
4
|
+
|
|
5
|
+
from cognite.neat.issues import NeatWarning
|
|
6
|
+
|
|
7
|
+
T_Identifier = TypeVar("T_Identifier", bound=Hashable)
|
|
8
|
+
|
|
9
|
+
T_ReferenceIdentifier = TypeVar("T_ReferenceIdentifier", bound=Hashable)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass(frozen=True)
|
|
13
|
+
class ResourceWarning(NeatWarning, Generic[T_Identifier]):
|
|
14
|
+
"""Base class for resource warnings"""
|
|
15
|
+
|
|
16
|
+
identifier: T_Identifier
|
|
17
|
+
resource_type: str
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass(frozen=True)
|
|
21
|
+
class ResourceNotFoundWarning(ResourceWarning[T_Identifier]):
|
|
22
|
+
"""The {resource_type} with identifier {identifier} is missing: {reason}. This will be ignored."""
|
|
23
|
+
|
|
24
|
+
fix = "Check the {resource_type} {identifier} and try again."
|
|
25
|
+
reason: str
|
|
26
|
+
|
|
27
|
+
def message(self) -> str:
|
|
28
|
+
return (self.__doc__ or "").format(
|
|
29
|
+
resource_type=self.resource_type, identifier=repr(self.identifier), reason=self.reason
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
def dump(self) -> dict[str, Any]:
|
|
33
|
+
output = super().dump()
|
|
34
|
+
output["resource_type"] = self.resource_type
|
|
35
|
+
output["identifier"] = self.identifier
|
|
36
|
+
output["reason"] = self.reason
|
|
37
|
+
return output
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@dataclass(frozen=True)
|
|
41
|
+
class ReferredResourceNotFoundWarning(ResourceWarning, Generic[T_Identifier, T_ReferenceIdentifier]):
|
|
42
|
+
"""The {resource_type} with identifier {identifier} referred by {referred_type} {referred_by} does not exist.
|
|
43
|
+
This will be ignored."""
|
|
44
|
+
|
|
45
|
+
fix = "Create the {resource_type}"
|
|
46
|
+
|
|
47
|
+
referred_by: T_ReferenceIdentifier
|
|
48
|
+
referred_type: str
|
|
49
|
+
|
|
50
|
+
def message(self) -> str:
|
|
51
|
+
return (self.__doc__ or "").format(
|
|
52
|
+
resource_type=self.resource_type,
|
|
53
|
+
identifier=repr(self.identifier),
|
|
54
|
+
referred_type=self.referred_type,
|
|
55
|
+
referred_by=repr(self.referred_by),
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
def dump(self) -> dict[str, Any]:
|
|
59
|
+
output = super().dump()
|
|
60
|
+
output["resource_type"] = self.resource_type
|
|
61
|
+
output["identifier"] = self.identifier
|
|
62
|
+
output["referred_by"] = self.referred_by
|
|
63
|
+
output["referred_type"] = self.referred_type
|
|
64
|
+
return output
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@dataclass(frozen=True)
|
|
68
|
+
class MultipleResourcesWarning(NeatWarning, Generic[T_Identifier]):
|
|
69
|
+
"""Multiple resources of type {resource_type} with identifiers {resources} were found. This will be ignored."""
|
|
70
|
+
|
|
71
|
+
fix = "Remove the duplicate resources"
|
|
72
|
+
|
|
73
|
+
resources: frozenset[T_Identifier]
|
|
74
|
+
resource_type: str
|
|
75
|
+
|
|
76
|
+
def message(self) -> str:
|
|
77
|
+
return (self.__doc__ or "").format(
|
|
78
|
+
resource_type=self.resource_type,
|
|
79
|
+
resources=self.resources,
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
def dump(self) -> dict[str, Any]:
|
|
83
|
+
output = super().dump()
|
|
84
|
+
output["resource_type"] = self.resource_type
|
|
85
|
+
output["resources"] = self.resources
|
|
86
|
+
return output
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
@dataclass(frozen=True)
|
|
90
|
+
class FailedLoadingResourcesWarning(NeatWarning, Generic[T_Identifier]):
|
|
91
|
+
"""Failed to load resources of type {resource_type} with identifiers {resources}. Continuing without
|
|
92
|
+
these resources."""
|
|
93
|
+
|
|
94
|
+
extra = "The error was: {error}"
|
|
95
|
+
|
|
96
|
+
fix = "Check the error."
|
|
97
|
+
|
|
98
|
+
resources: frozenset[T_Identifier]
|
|
99
|
+
resource_type: str
|
|
100
|
+
error: str | None = None
|
|
101
|
+
|
|
102
|
+
def message(self) -> str:
|
|
103
|
+
return (self.__doc__ or "").format(
|
|
104
|
+
resource_type=self.resource_type,
|
|
105
|
+
resources=self.resources,
|
|
106
|
+
) + (self.extra.format(error=self.error) if self.error else "")
|
|
107
|
+
|
|
108
|
+
def dump(self) -> dict[str, Any]:
|
|
109
|
+
output = super().dump()
|
|
110
|
+
output["resource_type"] = self.resource_type
|
|
111
|
+
output["resources"] = self.resources
|
|
112
|
+
return output
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
class ResourceTypeNotSupportedWarning(ResourceWarning[T_Identifier]):
|
|
116
|
+
"""The {resource_type} with identifier {identifier} is not supported. This will be ignored."""
|
|
117
|
+
|
|
118
|
+
def message(self) -> str:
|
|
119
|
+
return (self.__doc__ or "").format(resource_type=self.resource_type, identifier=repr(self.identifier))
|
|
120
|
+
|
|
121
|
+
def dump(self) -> dict[str, Any]:
|
|
122
|
+
output = super().dump()
|
|
123
|
+
output["resource_type"] = self.resource_type
|
|
124
|
+
output["identifier"] = self.identifier
|
|
125
|
+
return output
|
|
@@ -15,9 +15,10 @@ from cognite.client.data_classes.data_modeling import (
|
|
|
15
15
|
)
|
|
16
16
|
from cognite.client.exceptions import CogniteAPIError
|
|
17
17
|
|
|
18
|
+
from cognite.neat.issues import IssueList
|
|
19
|
+
from cognite.neat.issues.neat_warnings.resources import FailedLoadingResourcesWarning
|
|
18
20
|
from cognite.neat.rules import issues
|
|
19
21
|
from cognite.neat.rules._shared import Rules
|
|
20
|
-
from cognite.neat.rules.issues import IssueList
|
|
21
22
|
from cognite.neat.rules.models import InformationRules
|
|
22
23
|
from cognite.neat.rules.models.dms import DMSRules, DMSSchema, PipelineSchema
|
|
23
24
|
from cognite.neat.utils.cdf.loaders import (
|
|
@@ -313,7 +314,7 @@ class DMSExporter(CDFExporter[DMSSchema]):
|
|
|
313
314
|
try:
|
|
314
315
|
data_models = loader.client.data_modeling.data_models.list(space=space, limit=25, all_versions=False)
|
|
315
316
|
except CogniteAPIError as e:
|
|
316
|
-
warnings.warn(
|
|
317
|
+
warnings.warn(FailedLoadingResourcesWarning[str](frozenset({space}), "Space", str(e)), stacklevel=2)
|
|
317
318
|
return []
|
|
318
319
|
else:
|
|
319
320
|
return [
|
|
@@ -2,8 +2,8 @@ import warnings
|
|
|
2
2
|
from typing import Literal, overload
|
|
3
3
|
|
|
4
4
|
from cognite.neat.exceptions import wrangle_warnings
|
|
5
|
+
from cognite.neat.issues.neat_warnings.properties import PropertyRedefinedWarning
|
|
5
6
|
from cognite.neat.rules.issues.dms import EntityIDNotDMSCompliantWarning
|
|
6
|
-
from cognite.neat.rules.issues.importing import PropertyRedefinedWarning
|
|
7
7
|
from cognite.neat.rules.models import InformationRules
|
|
8
8
|
from cognite.neat.utils.regex_patterns import DMS_PROPERTY_ID_COMPLIANCE_REGEX, PATTERNS, VIEW_ID_COMPLIANCE_REGEX
|
|
9
9
|
|
|
@@ -78,7 +78,7 @@ def are_properties_redefined(rules: InformationRules, return_report: bool = Fals
|
|
|
78
78
|
elif property_.class_ in analyzed_properties[property_.property_]:
|
|
79
79
|
flag = True
|
|
80
80
|
warnings.warn(
|
|
81
|
-
PropertyRedefinedWarning(property_.
|
|
81
|
+
PropertyRedefinedWarning[str](property_.class_.versioned_id, "Class", property_.property_),
|
|
82
82
|
stacklevel=2,
|
|
83
83
|
)
|
|
84
84
|
|
|
@@ -1,14 +1,16 @@
|
|
|
1
1
|
from ._base import BaseImporter
|
|
2
2
|
from ._dms2rules import DMSImporter
|
|
3
3
|
from ._dtdl2rules import DTDLImporter
|
|
4
|
-
from .
|
|
5
|
-
from .
|
|
4
|
+
from ._rdf._imf2rules import IMFImporter
|
|
5
|
+
from ._rdf._inference2rules import InferenceImporter
|
|
6
|
+
from ._rdf._owl2rules import OWLImporter
|
|
6
7
|
from ._spreadsheet2rules import ExcelImporter, GoogleSheetImporter
|
|
7
8
|
from ._yaml2rules import YAMLImporter
|
|
8
9
|
|
|
9
10
|
__all__ = [
|
|
10
11
|
"BaseImporter",
|
|
11
12
|
"OWLImporter",
|
|
13
|
+
"IMFImporter",
|
|
12
14
|
"DMSImporter",
|
|
13
15
|
"ExcelImporter",
|
|
14
16
|
"GoogleSheetImporter",
|
|
@@ -25,7 +27,9 @@ def _repr_html_() -> str:
|
|
|
25
27
|
[
|
|
26
28
|
{
|
|
27
29
|
"Importer": name,
|
|
28
|
-
"Description":
|
|
30
|
+
"Description": (
|
|
31
|
+
globals()[name].__doc__.strip().split("\n")[0] if globals()[name].__doc__ else "Missing"
|
|
32
|
+
),
|
|
29
33
|
}
|
|
30
34
|
for name in __all__
|
|
31
35
|
if name != "BaseImporter"
|
|
@@ -9,9 +9,9 @@ from typing import Any, Literal, overload
|
|
|
9
9
|
from pydantic import ValidationError
|
|
10
10
|
from rdflib import Namespace
|
|
11
11
|
|
|
12
|
+
from cognite.neat.issues import IssueList, NeatError, NeatWarning
|
|
12
13
|
from cognite.neat.rules._shared import Rules
|
|
13
14
|
from cognite.neat.rules.issues.base import (
|
|
14
|
-
IssueList,
|
|
15
15
|
NeatValidationError,
|
|
16
16
|
ValidationWarning,
|
|
17
17
|
)
|
|
@@ -103,8 +103,8 @@ class _FutureResult:
|
|
|
103
103
|
@contextmanager
|
|
104
104
|
def _handle_issues(
|
|
105
105
|
issues: IssueList,
|
|
106
|
-
error_cls: type[
|
|
107
|
-
warning_cls: type[
|
|
106
|
+
error_cls: type[NeatError] = NeatValidationError,
|
|
107
|
+
warning_cls: type[NeatWarning] = ValidationWarning,
|
|
108
108
|
error_args: dict[str, Any] | None = None,
|
|
109
109
|
) -> Iterator[_FutureResult]:
|
|
110
110
|
"""This is an internal help function to handle issues and warnings.
|
|
@@ -18,9 +18,15 @@ from cognite.client.data_classes.data_modeling.views import (
|
|
|
18
18
|
)
|
|
19
19
|
from cognite.client.utils import ms_to_datetime
|
|
20
20
|
|
|
21
|
+
from cognite.neat.issues import IssueList, NeatIssue
|
|
22
|
+
from cognite.neat.issues.errors.resources import ResourceNotFoundError
|
|
23
|
+
from cognite.neat.issues.neat_warnings.properties import (
|
|
24
|
+
PropertyTypeNotSupportedWarning,
|
|
25
|
+
ReferredPropertyNotFoundWarning,
|
|
26
|
+
)
|
|
27
|
+
from cognite.neat.issues.neat_warnings.resources import ReferredResourceNotFoundWarning
|
|
21
28
|
from cognite.neat.rules import issues
|
|
22
29
|
from cognite.neat.rules.importers._base import BaseImporter, Rules, _handle_issues
|
|
23
|
-
from cognite.neat.rules.issues import IssueList, ValidationIssue
|
|
24
30
|
from cognite.neat.rules.models import (
|
|
25
31
|
DataModelType,
|
|
26
32
|
DMSRules,
|
|
@@ -60,7 +66,7 @@ class DMSImporter(BaseImporter):
|
|
|
60
66
|
def __init__(
|
|
61
67
|
self,
|
|
62
68
|
schema: DMSSchema,
|
|
63
|
-
read_issues: Sequence[
|
|
69
|
+
read_issues: Sequence[NeatIssue] | None = None,
|
|
64
70
|
metadata: DMSMetadata | None = None,
|
|
65
71
|
ref_metadata: DMSMetadata | None = None,
|
|
66
72
|
):
|
|
@@ -100,14 +106,28 @@ class DMSImporter(BaseImporter):
|
|
|
100
106
|
|
|
101
107
|
user_models = cls._find_model_in_list(data_models, data_model_id)
|
|
102
108
|
if len(user_models) == 0:
|
|
103
|
-
return cls(
|
|
109
|
+
return cls(
|
|
110
|
+
DMSSchema(),
|
|
111
|
+
[
|
|
112
|
+
ResourceNotFoundError[dm.DataModelId](
|
|
113
|
+
dm.DataModelId.load(reference_model_id), # type: ignore[arg-type]
|
|
114
|
+
"DataModel",
|
|
115
|
+
"Data Model is missing in CDF",
|
|
116
|
+
)
|
|
117
|
+
],
|
|
118
|
+
)
|
|
104
119
|
user_model = user_models.latest_version()
|
|
105
120
|
|
|
106
121
|
if reference_model_id:
|
|
107
122
|
ref_models = cls._find_model_in_list(data_models, reference_model_id)
|
|
108
123
|
if len(ref_models) == 0:
|
|
109
124
|
return cls(
|
|
110
|
-
DMSSchema(),
|
|
125
|
+
DMSSchema(),
|
|
126
|
+
[
|
|
127
|
+
ResourceNotFoundError[dm.DataModelId](
|
|
128
|
+
dm.DataModelId.load(reference_model_id), "DataModel", "Data Model is missing in CDF"
|
|
129
|
+
)
|
|
130
|
+
],
|
|
111
131
|
)
|
|
112
132
|
ref_model: dm.DataModel[dm.View] | None = ref_models.latest_version()
|
|
113
133
|
else:
|
|
@@ -200,7 +220,7 @@ class DMSImporter(BaseImporter):
|
|
|
200
220
|
return self._return_or_raise(self.issue_list, errors)
|
|
201
221
|
|
|
202
222
|
if not self.root_schema.data_model:
|
|
203
|
-
self.issue_list.append(
|
|
223
|
+
self.issue_list.append(ResourceNotFoundError[str]("Unknown", "DataModel", "Identifier is missing"))
|
|
204
224
|
return self._return_or_raise(self.issue_list, errors)
|
|
205
225
|
model = self.root_schema.data_model
|
|
206
226
|
with _handle_issues(
|
|
@@ -301,10 +321,11 @@ class DMSImporter(BaseImporter):
|
|
|
301
321
|
) -> DMSProperty | None:
|
|
302
322
|
if isinstance(prop, dm.MappedPropertyApply) and prop.container not in self._all_containers_by_id:
|
|
303
323
|
self.issue_list.append(
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
324
|
+
ReferredResourceNotFoundWarning[dm.ContainerId, dm.PropertyId](
|
|
325
|
+
dm.ContainerId.load(prop.container),
|
|
326
|
+
"Container",
|
|
327
|
+
view_entity.to_property_id(prop_id),
|
|
328
|
+
"View Property",
|
|
308
329
|
)
|
|
309
330
|
)
|
|
310
331
|
return None
|
|
@@ -313,11 +334,9 @@ class DMSImporter(BaseImporter):
|
|
|
313
334
|
and prop.container_property_identifier not in self._all_containers_by_id[prop.container].properties
|
|
314
335
|
):
|
|
315
336
|
self.issue_list.append(
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
container_id=str(ContainerEntity.from_id(prop.container)),
|
|
320
|
-
)
|
|
337
|
+
ReferredPropertyNotFoundWarning[dm.ContainerId, dm.ViewId](
|
|
338
|
+
prop.container, "Container", view_entity.as_id(), "View", prop_id
|
|
339
|
+
),
|
|
321
340
|
)
|
|
322
341
|
return None
|
|
323
342
|
if not isinstance(
|
|
@@ -329,7 +348,7 @@ class DMSImporter(BaseImporter):
|
|
|
329
348
|
| MultiReverseDirectRelationApply,
|
|
330
349
|
):
|
|
331
350
|
self.issue_list.append(
|
|
332
|
-
|
|
351
|
+
PropertyTypeNotSupportedWarning[dm.ViewId](view_entity.as_id(), "View", prop_id, type(prop).__name__)
|
|
333
352
|
)
|
|
334
353
|
return None
|
|
335
354
|
|
|
@@ -394,7 +413,9 @@ class DMSImporter(BaseImporter):
|
|
|
394
413
|
else:
|
|
395
414
|
return DataType.load(container_prop.type._type)
|
|
396
415
|
else:
|
|
397
|
-
self.issue_list.append(
|
|
416
|
+
self.issue_list.append(
|
|
417
|
+
PropertyTypeNotSupportedWarning[dm.ViewId](view_entity.as_id(), "View", prop_id, type(prop).__name__)
|
|
418
|
+
)
|
|
398
419
|
return None
|
|
399
420
|
|
|
400
421
|
def _get_nullable(self, prop: ViewPropertyApply) -> bool | None:
|
|
@@ -453,8 +474,8 @@ class DMSImporter(BaseImporter):
|
|
|
453
474
|
continue
|
|
454
475
|
else:
|
|
455
476
|
self.issue_list.append(
|
|
456
|
-
|
|
457
|
-
|
|
477
|
+
PropertyTypeNotSupportedWarning[dm.ContainerId](
|
|
478
|
+
prop.container, "Container", prop_id, type(constraint_obj).__name__
|
|
458
479
|
)
|
|
459
480
|
)
|
|
460
481
|
return unique_constraints or None
|
|
@@ -1,14 +1,18 @@
|
|
|
1
1
|
from collections import Counter
|
|
2
2
|
from collections.abc import Callable, Sequence
|
|
3
3
|
|
|
4
|
-
|
|
5
|
-
from cognite.neat.
|
|
4
|
+
from cognite.neat.issues import IssueList, NeatIssue
|
|
5
|
+
from cognite.neat.issues.errors.properties import PropertyTypeNotSupportedError
|
|
6
|
+
from cognite.neat.issues.errors.resources import MissingIdentifierError, ResourceNotFoundError
|
|
7
|
+
from cognite.neat.issues.neat_warnings.properties import PropertyTypeNotSupportedWarning
|
|
8
|
+
from cognite.neat.issues.neat_warnings.resources import ResourceTypeNotSupportedWarning
|
|
6
9
|
from cognite.neat.rules.importers._dtdl2rules.spec import (
|
|
7
10
|
DTMI,
|
|
8
11
|
Command,
|
|
9
12
|
CommandV2,
|
|
10
13
|
Component,
|
|
11
14
|
DTDLBase,
|
|
15
|
+
DTDLBaseWithName,
|
|
12
16
|
Enum,
|
|
13
17
|
Interface,
|
|
14
18
|
Object,
|
|
@@ -19,15 +23,14 @@ from cognite.neat.rules.importers._dtdl2rules.spec import (
|
|
|
19
23
|
Telemetry,
|
|
20
24
|
TelemetryV2,
|
|
21
25
|
)
|
|
22
|
-
from cognite.neat.rules.issues import IssueList, ValidationIssue
|
|
23
26
|
from cognite.neat.rules.models.data_types import _DATA_TYPE_BY_NAME, DataType, Json, String
|
|
24
27
|
from cognite.neat.rules.models.entities import ClassEntity
|
|
25
28
|
from cognite.neat.rules.models.information import InformationClass, InformationProperty
|
|
26
29
|
|
|
27
30
|
|
|
28
31
|
class _DTDLConverter:
|
|
29
|
-
def __init__(self, issues: list[
|
|
30
|
-
self.issues
|
|
32
|
+
def __init__(self, issues: list[NeatIssue] | None = None) -> None:
|
|
33
|
+
self.issues = IssueList(issues or [])
|
|
31
34
|
self.properties: list[InformationProperty] = []
|
|
32
35
|
self.classes: list[InformationClass] = []
|
|
33
36
|
self._item_by_id: dict[DTMI, DTDLBase] = {}
|
|
@@ -75,11 +78,10 @@ class _DTDLConverter:
|
|
|
75
78
|
convert_method(item, parent)
|
|
76
79
|
else:
|
|
77
80
|
self.issues.append(
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
)
|
|
81
|
+
ResourceTypeNotSupportedWarning[str](
|
|
82
|
+
item.id_.model_dump() if item.id_ else item.display_name or "missing",
|
|
83
|
+
item.type,
|
|
84
|
+
),
|
|
83
85
|
)
|
|
84
86
|
|
|
85
87
|
def convert_interface(self, item: Interface, _: str | None) -> None:
|
|
@@ -94,11 +96,11 @@ class _DTDLConverter:
|
|
|
94
96
|
for sub_item_or_id in item.contents or []:
|
|
95
97
|
if isinstance(sub_item_or_id, DTMI) and sub_item_or_id not in self._item_by_id:
|
|
96
98
|
self.issues.append(
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
99
|
+
PropertyTypeNotSupportedWarning(
|
|
100
|
+
item.id_.model_dump() or item.display_name or "missing",
|
|
101
|
+
item.type,
|
|
102
|
+
sub_item_or_id.path[-1],
|
|
103
|
+
".".join(sub_item_or_id.path),
|
|
102
104
|
)
|
|
103
105
|
)
|
|
104
106
|
elif isinstance(sub_item_or_id, DTMI):
|
|
@@ -130,12 +132,10 @@ class _DTDLConverter:
|
|
|
130
132
|
)
|
|
131
133
|
self.properties.append(prop)
|
|
132
134
|
|
|
133
|
-
def _missing_parent_warning(self, item):
|
|
135
|
+
def _missing_parent_warning(self, item: DTDLBaseWithName):
|
|
134
136
|
self.issues.append(
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
instance_name=item.display_name,
|
|
138
|
-
instance_id=item.id_.model_dump() if item.id_ else None,
|
|
137
|
+
ResourceNotFoundError[str](
|
|
138
|
+
(item.id_.model_dump() if item.id_ else item.display_name) or "missing", item.type, "parent missing"
|
|
139
139
|
)
|
|
140
140
|
)
|
|
141
141
|
|
|
@@ -151,22 +151,15 @@ class _DTDLConverter:
|
|
|
151
151
|
return None
|
|
152
152
|
if item.request is None:
|
|
153
153
|
self.issues.append(
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
instance_id=item.id_.model_dump() if item.id_ else None,
|
|
159
|
-
)
|
|
154
|
+
ResourceTypeNotSupportedWarning[str](
|
|
155
|
+
item.id_.model_dump() if item.id_ else item.display_name or "missing",
|
|
156
|
+
f"{item.type}.request",
|
|
157
|
+
),
|
|
160
158
|
)
|
|
161
159
|
return None
|
|
162
160
|
if item.response is not None:
|
|
163
161
|
# Currently, we do not know how to handle response
|
|
164
|
-
self.issues.append(
|
|
165
|
-
issues.importing.IgnoredComponentWarning(
|
|
166
|
-
identifier=f"{parent}.response",
|
|
167
|
-
reason="Neat does not have a concept of response for commands. This will be ignored.",
|
|
168
|
-
)
|
|
169
|
-
)
|
|
162
|
+
self.issues.append(ResourceTypeNotSupportedWarning[str](f"{parent}.response", "Command.Response"))
|
|
170
163
|
value_type = self.schema_to_value_type(item.request.schema_, item)
|
|
171
164
|
if value_type is None:
|
|
172
165
|
return
|
|
@@ -213,10 +206,9 @@ class _DTDLConverter:
|
|
|
213
206
|
else:
|
|
214
207
|
# Falling back to json
|
|
215
208
|
self.issues.append(
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
instance_id=item.target.model_dump(),
|
|
209
|
+
MissingIdentifierError(
|
|
210
|
+
"Unknown",
|
|
211
|
+
item.target.model_dump(),
|
|
220
212
|
)
|
|
221
213
|
)
|
|
222
214
|
value_type = Json()
|
|
@@ -239,9 +231,9 @@ class _DTDLConverter:
|
|
|
239
231
|
def convert_object(self, item: Object, _: str | None) -> None:
|
|
240
232
|
if item.id_ is None:
|
|
241
233
|
self.issues.append(
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
234
|
+
MissingIdentifierError(
|
|
235
|
+
resource_type=item.type,
|
|
236
|
+
name=item.display_name,
|
|
245
237
|
)
|
|
246
238
|
)
|
|
247
239
|
return None
|
|
@@ -280,21 +272,20 @@ class _DTDLConverter:
|
|
|
280
272
|
return _DATA_TYPE_BY_NAME[input_type.casefold()]()
|
|
281
273
|
elif isinstance(input_type, str):
|
|
282
274
|
self.issues.append(
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
instance_id=item.id_.model_dump() if item.id_ else None,
|
|
275
|
+
PropertyTypeNotSupportedError[str](
|
|
276
|
+
(item.id_.model_dump() if item.id_ else item.display_name) or "missing",
|
|
277
|
+
item.type,
|
|
278
|
+
"schema",
|
|
279
|
+
input_type,
|
|
289
280
|
)
|
|
290
281
|
)
|
|
291
282
|
return None
|
|
292
283
|
elif isinstance(input_type, Object | Interface):
|
|
293
284
|
if input_type.id_ is None:
|
|
294
285
|
self.issues.append(
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
286
|
+
MissingIdentifierError(
|
|
287
|
+
input_type.type,
|
|
288
|
+
input_type.display_name,
|
|
298
289
|
)
|
|
299
290
|
)
|
|
300
291
|
return Json()
|
|
@@ -304,11 +295,11 @@ class _DTDLConverter:
|
|
|
304
295
|
return ClassEntity.load(input_type.id_.as_class_id())
|
|
305
296
|
else:
|
|
306
297
|
self.issues.append(
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
298
|
+
PropertyTypeNotSupportedWarning(
|
|
299
|
+
item.id_.model_dump() if item.id_ else item.display_name or "missing",
|
|
300
|
+
item.type,
|
|
301
|
+
"schema",
|
|
302
|
+
input_type.type if input_type else "missing",
|
|
312
303
|
)
|
|
313
304
|
)
|
|
314
305
|
return None
|
|
@@ -6,12 +6,13 @@ from typing import Literal, overload
|
|
|
6
6
|
|
|
7
7
|
from pydantic import ValidationError
|
|
8
8
|
|
|
9
|
+
from cognite.neat.issues import IssueList, NeatIssue
|
|
9
10
|
from cognite.neat.rules import issues
|
|
10
11
|
from cognite.neat.rules._shared import Rules
|
|
11
12
|
from cognite.neat.rules.importers._base import BaseImporter, _handle_issues
|
|
12
13
|
from cognite.neat.rules.importers._dtdl2rules.dtdl_converter import _DTDLConverter
|
|
13
14
|
from cognite.neat.rules.importers._dtdl2rules.spec import DTDL_CLS_BY_TYPE_BY_SPEC, DTDLBase, Interface
|
|
14
|
-
from cognite.neat.rules.issues import
|
|
15
|
+
from cognite.neat.rules.issues import ValidationIssue
|
|
15
16
|
from cognite.neat.rules.models import InformationRules, RoleTypes, SchemaCompleteness, SheetList
|
|
16
17
|
from cognite.neat.rules.models.information import InformationClass, InformationProperty
|
|
17
18
|
from cognite.neat.utils.text import to_pascal
|
|
@@ -36,7 +37,7 @@ class DTDLImporter(BaseImporter):
|
|
|
36
37
|
self,
|
|
37
38
|
items: Sequence[DTDLBase],
|
|
38
39
|
title: str | None = None,
|
|
39
|
-
read_issues: list[
|
|
40
|
+
read_issues: list[NeatIssue] | None = None,
|
|
40
41
|
schema: SchemaCompleteness = SchemaCompleteness.partial,
|
|
41
42
|
) -> None:
|
|
42
43
|
self._items = items
|
|
@@ -94,10 +95,10 @@ class DTDLImporter(BaseImporter):
|
|
|
94
95
|
@classmethod
|
|
95
96
|
def from_directory(cls, directory: Path) -> "DTDLImporter":
|
|
96
97
|
items: list[DTDLBase] = []
|
|
97
|
-
issues: list[
|
|
98
|
+
issues: list[NeatIssue] = []
|
|
98
99
|
for filepath in directory.glob("**/*.json"):
|
|
99
100
|
for item in cls._from_file_content(filepath.read_text(), filepath):
|
|
100
|
-
if isinstance(item,
|
|
101
|
+
if isinstance(item, NeatIssue):
|
|
101
102
|
issues.append(item)
|
|
102
103
|
else:
|
|
103
104
|
items.append(item)
|
|
@@ -106,7 +107,7 @@ class DTDLImporter(BaseImporter):
|
|
|
106
107
|
@classmethod
|
|
107
108
|
def from_zip(cls, zip_file: Path) -> "DTDLImporter":
|
|
108
109
|
items: list[DTDLBase] = []
|
|
109
|
-
issues: list[
|
|
110
|
+
issues: list[NeatIssue] = []
|
|
110
111
|
with zipfile.ZipFile(zip_file) as z:
|
|
111
112
|
for filepath in z.namelist():
|
|
112
113
|
if filepath.endswith(".json"):
|
|
File without changes
|