cognite-neat 0.109.4__py3-none-any.whl → 0.111.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_alpha.py +8 -0
- cognite/neat/_client/_api/schema.py +43 -1
- cognite/neat/_client/data_classes/schema.py +4 -4
- cognite/neat/_constants.py +15 -1
- cognite/neat/_graph/extractors/__init__.py +4 -0
- cognite/neat/_graph/extractors/_classic_cdf/_base.py +8 -16
- cognite/neat/_graph/extractors/_classic_cdf/_classic.py +48 -19
- cognite/neat/_graph/extractors/_classic_cdf/_relationships.py +23 -17
- cognite/neat/_graph/extractors/_classic_cdf/_sequences.py +15 -17
- cognite/neat/_graph/extractors/_dict.py +102 -0
- cognite/neat/_graph/extractors/_dms.py +27 -40
- cognite/neat/_graph/extractors/_dms_graph.py +30 -3
- cognite/neat/_graph/extractors/_iodd.py +3 -3
- cognite/neat/_graph/extractors/_mock_graph_generator.py +9 -7
- cognite/neat/_graph/extractors/_raw.py +67 -0
- cognite/neat/_graph/loaders/_base.py +20 -4
- cognite/neat/_graph/loaders/_rdf2dms.py +476 -383
- cognite/neat/_graph/queries/_base.py +163 -133
- cognite/neat/_graph/transformers/__init__.py +1 -3
- cognite/neat/_graph/transformers/_classic_cdf.py +6 -22
- cognite/neat/_graph/transformers/_rdfpath.py +2 -49
- cognite/neat/_issues/__init__.py +1 -6
- cognite/neat/_issues/_base.py +21 -252
- cognite/neat/_issues/_contextmanagers.py +46 -0
- cognite/neat/_issues/_factory.py +69 -0
- cognite/neat/_issues/errors/__init__.py +20 -4
- cognite/neat/_issues/errors/_external.py +7 -0
- cognite/neat/_issues/errors/_wrapper.py +81 -3
- cognite/neat/_issues/formatters.py +4 -4
- cognite/neat/_issues/warnings/__init__.py +3 -2
- cognite/neat/_issues/warnings/_properties.py +8 -0
- cognite/neat/_issues/warnings/user_modeling.py +12 -0
- cognite/neat/_rules/_constants.py +12 -0
- cognite/neat/_rules/_shared.py +3 -2
- cognite/neat/_rules/analysis/__init__.py +2 -3
- cognite/neat/_rules/analysis/_base.py +430 -259
- cognite/neat/_rules/catalog/info-rules-imf.xlsx +0 -0
- cognite/neat/_rules/exporters/_rules2excel.py +3 -9
- cognite/neat/_rules/exporters/_rules2instance_template.py +2 -2
- cognite/neat/_rules/exporters/_rules2ontology.py +5 -4
- cognite/neat/_rules/importers/_base.py +2 -47
- cognite/neat/_rules/importers/_dms2rules.py +7 -10
- cognite/neat/_rules/importers/_dtdl2rules/dtdl_importer.py +2 -2
- cognite/neat/_rules/importers/_rdf/_inference2rules.py +66 -26
- cognite/neat/_rules/importers/_rdf/_shared.py +1 -1
- cognite/neat/_rules/importers/_spreadsheet2rules.py +12 -9
- cognite/neat/_rules/models/_base_rules.py +0 -2
- cognite/neat/_rules/models/data_types.py +7 -0
- cognite/neat/_rules/models/dms/_exporter.py +9 -8
- cognite/neat/_rules/models/dms/_rules.py +29 -2
- cognite/neat/_rules/models/dms/_rules_input.py +9 -1
- cognite/neat/_rules/models/dms/_validation.py +115 -5
- cognite/neat/_rules/models/entities/_loaders.py +1 -1
- cognite/neat/_rules/models/entities/_multi_value.py +2 -2
- cognite/neat/_rules/models/entities/_single_value.py +8 -3
- cognite/neat/_rules/models/entities/_wrapped.py +2 -2
- cognite/neat/_rules/models/information/_rules.py +18 -17
- cognite/neat/_rules/models/information/_rules_input.py +3 -1
- cognite/neat/_rules/models/information/_validation.py +66 -17
- cognite/neat/_rules/transformers/__init__.py +8 -2
- cognite/neat/_rules/transformers/_converters.py +234 -44
- cognite/neat/_rules/transformers/_verification.py +5 -10
- cognite/neat/_session/_base.py +6 -4
- cognite/neat/_session/_explore.py +39 -0
- cognite/neat/_session/_inspect.py +25 -6
- cognite/neat/_session/_prepare.py +12 -0
- cognite/neat/_session/_read.py +88 -20
- cognite/neat/_session/_set.py +7 -1
- cognite/neat/_session/_show.py +11 -123
- cognite/neat/_session/_state.py +6 -2
- cognite/neat/_session/_subset.py +64 -0
- cognite/neat/_session/_to.py +177 -19
- cognite/neat/_store/_graph_store.py +9 -246
- cognite/neat/_utils/rdf_.py +36 -5
- cognite/neat/_utils/spreadsheet.py +44 -1
- cognite/neat/_utils/text.py +124 -37
- cognite/neat/_utils/upload.py +2 -0
- cognite/neat/_version.py +2 -2
- {cognite_neat-0.109.4.dist-info → cognite_neat-0.111.0.dist-info}/METADATA +1 -1
- {cognite_neat-0.109.4.dist-info → cognite_neat-0.111.0.dist-info}/RECORD +83 -82
- {cognite_neat-0.109.4.dist-info → cognite_neat-0.111.0.dist-info}/WHEEL +1 -1
- cognite/neat/_graph/queries/_construct.py +0 -187
- cognite/neat/_graph/queries/_shared.py +0 -173
- cognite/neat/_rules/analysis/_dms.py +0 -57
- cognite/neat/_rules/analysis/_information.py +0 -249
- cognite/neat/_rules/models/_rdfpath.py +0 -372
- {cognite_neat-0.109.4.dist-info → cognite_neat-0.111.0.dist-info}/LICENSE +0 -0
- {cognite_neat-0.109.4.dist-info → cognite_neat-0.111.0.dist-info}/entry_points.txt +0 -0
|
Binary file
|
|
@@ -13,6 +13,7 @@ from openpyxl.styles import Alignment, Border, Font, PatternFill, Side
|
|
|
13
13
|
from openpyxl.worksheet.worksheet import Worksheet
|
|
14
14
|
from rdflib import Namespace
|
|
15
15
|
|
|
16
|
+
from cognite.neat._rules._constants import get_internal_properties
|
|
16
17
|
from cognite.neat._rules._shared import VerifiedRules
|
|
17
18
|
from cognite.neat._rules.models import (
|
|
18
19
|
ExtensionCategory,
|
|
@@ -54,7 +55,7 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
|
|
|
54
55
|
Style = Literal["none", "minimal", "default", "maximal"]
|
|
55
56
|
DumpOptions = Literal["user", "last", "reference"]
|
|
56
57
|
_main_header_by_sheet_name: ClassVar[dict[str, str]] = {
|
|
57
|
-
"Properties": "Definition of Properties
|
|
58
|
+
"Properties": "Definition of Properties",
|
|
58
59
|
"Classes": "Definition of Classes",
|
|
59
60
|
"Views": "Definition of Views",
|
|
60
61
|
"Containers": "Definition of Containers",
|
|
@@ -64,13 +65,6 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
|
|
|
64
65
|
style_options = get_args(Style)
|
|
65
66
|
dump_options = get_args(DumpOptions)
|
|
66
67
|
|
|
67
|
-
_internal_columns: ClassVar[list[str]] = [
|
|
68
|
-
"physical",
|
|
69
|
-
"logical",
|
|
70
|
-
"conceptual",
|
|
71
|
-
"Neat ID",
|
|
72
|
-
]
|
|
73
|
-
|
|
74
68
|
def __init__(
|
|
75
69
|
self,
|
|
76
70
|
styling: Style = "default",
|
|
@@ -131,7 +125,7 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
|
|
|
131
125
|
if sheet.lower() == "metadata":
|
|
132
126
|
continue
|
|
133
127
|
ws = workbook[sheet]
|
|
134
|
-
for col in
|
|
128
|
+
for col in get_internal_properties():
|
|
135
129
|
column_letter = find_column_with_value(ws, col)
|
|
136
130
|
if column_letter:
|
|
137
131
|
ws.column_dimensions[column_letter].hidden = True
|
|
@@ -9,7 +9,7 @@ from openpyxl.utils import get_column_letter
|
|
|
9
9
|
from openpyxl.worksheet.datavalidation import DataValidation
|
|
10
10
|
|
|
11
11
|
from cognite.neat._rules._constants import EntityTypes
|
|
12
|
-
from cognite.neat._rules.analysis import
|
|
12
|
+
from cognite.neat._rules.analysis import RulesAnalysis
|
|
13
13
|
from cognite.neat._rules.models.entities._single_value import ClassEntity
|
|
14
14
|
from cognite.neat._rules.models.information._rules import InformationRules
|
|
15
15
|
|
|
@@ -55,7 +55,7 @@ class InstanceTemplateExporter(BaseExporter[InformationRules, Workbook]):
|
|
|
55
55
|
# Remove default sheet named "Sheet"
|
|
56
56
|
workbook.remove(workbook["Sheet"])
|
|
57
57
|
|
|
58
|
-
for class_, properties in
|
|
58
|
+
for class_, properties in RulesAnalysis(rules).properties_by_id_by_class().items():
|
|
59
59
|
workbook.create_sheet(title=class_.suffix)
|
|
60
60
|
|
|
61
61
|
# Add header rows
|
|
@@ -15,7 +15,7 @@ from cognite.neat._issues.errors import (
|
|
|
15
15
|
)
|
|
16
16
|
from cognite.neat._issues.warnings import PropertyDefinitionDuplicatedWarning
|
|
17
17
|
from cognite.neat._rules._constants import EntityTypes
|
|
18
|
-
from cognite.neat._rules.analysis import
|
|
18
|
+
from cognite.neat._rules.analysis import RulesAnalysis
|
|
19
19
|
from cognite.neat._rules.models.data_types import DataType
|
|
20
20
|
from cognite.neat._rules.models.entities import ClassEntity
|
|
21
21
|
from cognite.neat._rules.models.information import (
|
|
@@ -121,11 +121,12 @@ class Ontology(OntologyModel):
|
|
|
121
121
|
)
|
|
122
122
|
raise MultiValueError(errors)
|
|
123
123
|
|
|
124
|
-
|
|
124
|
+
analysis = RulesAnalysis(rules)
|
|
125
|
+
class_dict = analysis.class_by_suffix()
|
|
125
126
|
return cls(
|
|
126
127
|
properties=[
|
|
127
128
|
OWLProperty.from_list_of_properties(definition, rules.metadata.namespace)
|
|
128
|
-
for definition in
|
|
129
|
+
for definition in analysis.property_by_id().values()
|
|
129
130
|
],
|
|
130
131
|
classes=[
|
|
131
132
|
OWLClass.from_class(definition, rules.metadata.namespace, rules.prefixes)
|
|
@@ -137,7 +138,7 @@ class Ontology(OntologyModel):
|
|
|
137
138
|
list(properties.values()),
|
|
138
139
|
rules.metadata.namespace,
|
|
139
140
|
)
|
|
140
|
-
for class_, properties in
|
|
141
|
+
for class_, properties in analysis.properties_by_id_by_class().items()
|
|
141
142
|
]
|
|
142
143
|
+ [
|
|
143
144
|
SHACLNodeShape.from_rules(
|
|
@@ -1,15 +1,11 @@
|
|
|
1
|
-
import warnings
|
|
2
1
|
from abc import ABC, abstractmethod
|
|
3
|
-
from
|
|
4
|
-
from contextlib import contextmanager, suppress
|
|
2
|
+
from contextlib import suppress
|
|
5
3
|
from datetime import datetime
|
|
6
|
-
from typing import TYPE_CHECKING, Any, Generic
|
|
4
|
+
from typing import TYPE_CHECKING, Any, Generic
|
|
7
5
|
|
|
8
|
-
from pydantic import ValidationError
|
|
9
6
|
from rdflib import URIRef
|
|
10
7
|
|
|
11
8
|
from cognite.neat._constants import DEFAULT_NAMESPACE
|
|
12
|
-
from cognite.neat._issues import IssueList, NeatError, NeatWarning
|
|
13
9
|
from cognite.neat._rules._shared import ReadRules, T_InputRules
|
|
14
10
|
from cognite.neat._utils.auxiliary import class_html_doc
|
|
15
11
|
|
|
@@ -64,44 +60,3 @@ class BaseImporter(ABC, Generic[T_InputRules]):
|
|
|
64
60
|
@property
|
|
65
61
|
def source_uri(self) -> URIRef:
|
|
66
62
|
return DEFAULT_NAMESPACE["UNKNOWN"]
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
class _FutureResult:
|
|
70
|
-
def __init__(self) -> None:
|
|
71
|
-
self._result: Literal["success", "failure", "pending"] = "pending"
|
|
72
|
-
|
|
73
|
-
@property
|
|
74
|
-
def result(self) -> Literal["success", "failure", "pending"]:
|
|
75
|
-
return self._result
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
@contextmanager
|
|
79
|
-
def _handle_issues(
|
|
80
|
-
issues: IssueList,
|
|
81
|
-
error_cls: type[NeatError] = NeatError,
|
|
82
|
-
warning_cls: type[NeatWarning] = NeatWarning,
|
|
83
|
-
error_args: dict[str, Any] | None = None,
|
|
84
|
-
) -> Iterator[_FutureResult]:
|
|
85
|
-
"""This is an internal help function to handle issues and warnings.
|
|
86
|
-
|
|
87
|
-
Args:
|
|
88
|
-
issues: The issues list to append to.
|
|
89
|
-
error_cls: The class used to convert errors to issues.
|
|
90
|
-
warning_cls: The class used to convert warnings to issues.
|
|
91
|
-
|
|
92
|
-
Returns:
|
|
93
|
-
FutureResult: A future result object that can be used to check the result of the context manager.
|
|
94
|
-
"""
|
|
95
|
-
with warnings.catch_warnings(record=True) as warning_logger:
|
|
96
|
-
warnings.simplefilter("always")
|
|
97
|
-
future_result = _FutureResult()
|
|
98
|
-
try:
|
|
99
|
-
yield future_result
|
|
100
|
-
except ValidationError as e:
|
|
101
|
-
issues.extend(error_cls.from_errors(e.errors(), **(error_args or {}))) # type: ignore[arg-type]
|
|
102
|
-
future_result._result = "failure"
|
|
103
|
-
else:
|
|
104
|
-
future_result._result = "success"
|
|
105
|
-
finally:
|
|
106
|
-
if warning_logger:
|
|
107
|
-
issues.extend([warning_cls.from_warning(warning) for warning in warning_logger]) # type: ignore[misc]
|
|
@@ -19,7 +19,7 @@ from cognite.client.data_classes.data_modeling.views import (
|
|
|
19
19
|
from cognite.client.utils import ms_to_datetime
|
|
20
20
|
|
|
21
21
|
from cognite.neat._client import NeatClient
|
|
22
|
-
from cognite.neat._issues import IssueList, MultiValueError, NeatIssue
|
|
22
|
+
from cognite.neat._issues import IssueList, MultiValueError, NeatIssue, catch_issues
|
|
23
23
|
from cognite.neat._issues.errors import (
|
|
24
24
|
FileTypeUnexpectedError,
|
|
25
25
|
NeatValueError,
|
|
@@ -35,7 +35,7 @@ from cognite.neat._issues.warnings import (
|
|
|
35
35
|
ResourceUnknownWarning,
|
|
36
36
|
)
|
|
37
37
|
from cognite.neat._rules._shared import ReadRules
|
|
38
|
-
from cognite.neat._rules.importers._base import BaseImporter
|
|
38
|
+
from cognite.neat._rules.importers._base import BaseImporter
|
|
39
39
|
from cognite.neat._rules.models import (
|
|
40
40
|
DMSInputRules,
|
|
41
41
|
DMSSchema,
|
|
@@ -131,11 +131,10 @@ class DMSImporter(BaseImporter[DMSInputRules]):
|
|
|
131
131
|
|
|
132
132
|
@classmethod
|
|
133
133
|
def from_data_model(cls, client: NeatClient, model: dm.DataModel[dm.View]) -> "DMSImporter":
|
|
134
|
-
|
|
135
|
-
with _handle_issues(issue_list) as result:
|
|
134
|
+
with catch_issues() as issue_list:
|
|
136
135
|
schema = client.schema.retrieve_data_model(model)
|
|
137
136
|
|
|
138
|
-
if
|
|
137
|
+
if issue_list.has_errors:
|
|
139
138
|
return cls(DMSSchema(), issue_list)
|
|
140
139
|
|
|
141
140
|
metadata = cls._create_metadata_from_model(model)
|
|
@@ -187,10 +186,9 @@ class DMSImporter(BaseImporter[DMSInputRules]):
|
|
|
187
186
|
|
|
188
187
|
@classmethod
|
|
189
188
|
def from_directory(cls, directory: str | Path, client: NeatClient | None = None) -> "DMSImporter":
|
|
190
|
-
|
|
191
|
-
with _handle_issues(issue_list) as _:
|
|
189
|
+
with catch_issues() as issue_list:
|
|
192
190
|
schema = DMSSchema.from_directory(directory)
|
|
193
|
-
# If there were errors during the import, the to_rules
|
|
191
|
+
# If there were errors during the import, the to_rules will raise them.
|
|
194
192
|
return cls(
|
|
195
193
|
schema, issue_list, referenced_containers=cls._lookup_referenced_containers(schema, issue_list, client)
|
|
196
194
|
)
|
|
@@ -202,8 +200,7 @@ class DMSImporter(BaseImporter[DMSInputRules]):
|
|
|
202
200
|
DMSSchema(),
|
|
203
201
|
[FileTypeUnexpectedError(Path(zip_file), frozenset([".zip"]))],
|
|
204
202
|
)
|
|
205
|
-
|
|
206
|
-
with _handle_issues(issue_list) as _:
|
|
203
|
+
with catch_issues() as issue_list:
|
|
207
204
|
schema = DMSSchema.from_zip(zip_file)
|
|
208
205
|
return cls(
|
|
209
206
|
schema, issue_list, referenced_containers=cls._lookup_referenced_containers(schema, issue_list, client)
|
|
@@ -19,7 +19,7 @@ from cognite.neat._rules.importers._dtdl2rules.dtdl_converter import _DTDLConver
|
|
|
19
19
|
from cognite.neat._rules.importers._dtdl2rules.spec import DTDL_CLS_BY_TYPE_BY_SPEC, DTDLBase, Interface
|
|
20
20
|
from cognite.neat._rules.models import InformationInputRules
|
|
21
21
|
from cognite.neat._rules.models.information import InformationInputMetadata
|
|
22
|
-
from cognite.neat._utils.text import humanize_collection,
|
|
22
|
+
from cognite.neat._utils.text import humanize_collection, to_pascal_case
|
|
23
23
|
|
|
24
24
|
|
|
25
25
|
class DTDLImporter(BaseImporter[InformationInputRules]):
|
|
@@ -130,7 +130,7 @@ class DTDLImporter(BaseImporter[InformationInputRules]):
|
|
|
130
130
|
metadata = self._default_metadata()
|
|
131
131
|
|
|
132
132
|
if self.name:
|
|
133
|
-
metadata["name"] =
|
|
133
|
+
metadata["name"] = to_pascal_case(self.name)
|
|
134
134
|
try:
|
|
135
135
|
most_common_prefix = converter.get_most_common_prefix()
|
|
136
136
|
except ValueError:
|
|
@@ -10,14 +10,16 @@ from cognite.client import data_modeling as dm
|
|
|
10
10
|
from rdflib import RDF, RDFS, Graph, Namespace, URIRef
|
|
11
11
|
from rdflib import Literal as RdfLiteral
|
|
12
12
|
|
|
13
|
+
from cognite.neat._config import GLOBAL_CONFIG
|
|
13
14
|
from cognite.neat._constants import NEAT, get_default_prefixes_and_namespaces
|
|
14
15
|
from cognite.neat._issues import IssueList
|
|
15
16
|
from cognite.neat._issues.warnings import PropertyValueTypeUndefinedWarning
|
|
16
|
-
from cognite.neat._rules.analysis import
|
|
17
|
+
from cognite.neat._rules.analysis import RulesAnalysis
|
|
17
18
|
from cognite.neat._rules.models import InformationRules, data_types
|
|
18
19
|
from cognite.neat._rules.models.data_types import AnyURI
|
|
19
20
|
from cognite.neat._rules.models.entities._single_value import UnknownEntity
|
|
20
21
|
from cognite.neat._rules.models.information import (
|
|
22
|
+
InformationClass,
|
|
21
23
|
InformationInputClass,
|
|
22
24
|
InformationInputProperty,
|
|
23
25
|
InformationMetadata,
|
|
@@ -26,6 +28,7 @@ from cognite.neat._store import NeatGraphStore
|
|
|
26
28
|
from cognite.neat._store._provenance import INSTANCES_ENTITY
|
|
27
29
|
from cognite.neat._utils.collection_ import iterate_progress_bar
|
|
28
30
|
from cognite.neat._utils.rdf_ import remove_namespace_from_uri, uri_to_short_form
|
|
31
|
+
from cognite.neat._utils.text import NamingStandardization
|
|
29
32
|
|
|
30
33
|
from ._base import DEFAULT_NON_EXISTING_NODE_TYPE, BaseRDFImporter
|
|
31
34
|
|
|
@@ -256,7 +259,7 @@ class InferenceImporter(BaseRDFImporter):
|
|
|
256
259
|
property_["value_type"].remove(str(self.non_existing_node_type))
|
|
257
260
|
|
|
258
261
|
if len(property_["value_type"]) > 1:
|
|
259
|
-
property_["value_type"] = "
|
|
262
|
+
property_["value_type"] = ", ".join([str(t) for t in property_["value_type"]])
|
|
260
263
|
else:
|
|
261
264
|
property_["value_type"] = next(iter(property_["value_type"]))
|
|
262
265
|
|
|
@@ -402,7 +405,8 @@ class SubclassInferenceImporter(BaseRDFImporter):
|
|
|
402
405
|
else:
|
|
403
406
|
existing_classes = {}
|
|
404
407
|
classes: list[InformationInputClass] = []
|
|
405
|
-
|
|
408
|
+
properties_by_class_suffix_by_property_id: dict[str, dict[str, InformationInputProperty]] = {}
|
|
409
|
+
|
|
406
410
|
# Help for IDE
|
|
407
411
|
type_uri: URIRef
|
|
408
412
|
parent_uri: URIRef
|
|
@@ -440,26 +444,51 @@ class SubclassInferenceImporter(BaseRDFImporter):
|
|
|
440
444
|
InformationInputClass(
|
|
441
445
|
class_=class_suffix,
|
|
442
446
|
implements=parent_suffix,
|
|
447
|
+
instance_source=type_uri,
|
|
443
448
|
)
|
|
444
449
|
)
|
|
445
450
|
else:
|
|
446
451
|
classes.append(InformationInputClass.load(existing_classes[class_suffix].model_dump()))
|
|
452
|
+
|
|
453
|
+
properties_by_id: dict[str, InformationInputProperty] = {}
|
|
447
454
|
for property_uri, read_properties in properties_by_property_uri.items():
|
|
448
455
|
if property_uri in shared_property_uris:
|
|
449
456
|
shared_properties[property_uri].extend(read_properties)
|
|
450
457
|
continue
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
)
|
|
454
|
-
|
|
458
|
+
property_id = remove_namespace_from_uri(property_uri)
|
|
459
|
+
self._add_uri_namespace_to_prefixes(property_uri, prefixes)
|
|
460
|
+
property_id_standardized = NamingStandardization.standardize_property_str(property_uri)
|
|
461
|
+
if existing_prop := properties_by_id.get(property_id_standardized):
|
|
462
|
+
if not isinstance(existing_prop.instance_source, list):
|
|
463
|
+
existing_prop.instance_source = (
|
|
464
|
+
[existing_prop.instance_source] if existing_prop.instance_source else []
|
|
465
|
+
)
|
|
466
|
+
existing_prop.instance_source.append(property_uri)
|
|
467
|
+
continue
|
|
468
|
+
else:
|
|
469
|
+
properties_by_id[property_id_standardized] = self._create_property(
|
|
470
|
+
read_properties, class_suffix, property_uri, property_id, prefixes
|
|
471
|
+
)
|
|
472
|
+
properties_by_class_suffix_by_property_id[class_suffix] = properties_by_id
|
|
455
473
|
if parent_suffix:
|
|
474
|
+
properties_by_id = {}
|
|
456
475
|
for property_uri, read_properties in shared_properties.items():
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
476
|
+
property_id = remove_namespace_from_uri(property_uri)
|
|
477
|
+
self._add_uri_namespace_to_prefixes(property_uri, prefixes)
|
|
478
|
+
property_id_standardized = NamingStandardization.standardize_property_str(property_uri)
|
|
479
|
+
if existing_prop := properties_by_id.get(property_id_standardized):
|
|
480
|
+
if not isinstance(existing_prop.instance_source, list):
|
|
481
|
+
existing_prop.instance_source = (
|
|
482
|
+
[existing_prop.instance_source] if existing_prop.instance_source else []
|
|
483
|
+
)
|
|
484
|
+
existing_prop.instance_source.append(property_uri)
|
|
485
|
+
else:
|
|
486
|
+
properties_by_id[property_id_standardized] = self._create_property(
|
|
487
|
+
read_properties, parent_suffix, property_uri, property_id, prefixes
|
|
460
488
|
)
|
|
461
|
-
|
|
462
|
-
|
|
489
|
+
return classes, [
|
|
490
|
+
prop for properties in properties_by_class_suffix_by_property_id.values() for prop in properties.values()
|
|
491
|
+
]
|
|
463
492
|
|
|
464
493
|
@staticmethod
|
|
465
494
|
def _get_properties_by_class_by_property(
|
|
@@ -481,26 +510,40 @@ class SubclassInferenceImporter(BaseRDFImporter):
|
|
|
481
510
|
type_uri, instance_count_literal = cast(tuple[URIRef, RdfLiteral], result_row)
|
|
482
511
|
count_by_type[type_uri] = instance_count_literal.toPython()
|
|
483
512
|
if self._rules:
|
|
484
|
-
analysis =
|
|
513
|
+
analysis = RulesAnalysis(self._rules)
|
|
485
514
|
existing_class_properties = {
|
|
486
|
-
(class_entity.suffix, prop.property_)
|
|
487
|
-
for class_entity, properties in analysis.
|
|
488
|
-
|
|
515
|
+
(class_entity.suffix, prop.property_): prop
|
|
516
|
+
for class_entity, properties in analysis.properties_by_class(
|
|
517
|
+
include_ancestors=True, include_different_space=True
|
|
489
518
|
).items()
|
|
490
519
|
for prop in properties
|
|
491
520
|
}
|
|
521
|
+
existing_classes = {cls_.class_.suffix: cls_ for cls_ in self._rules.classes}
|
|
492
522
|
else:
|
|
493
|
-
existing_class_properties =
|
|
523
|
+
existing_class_properties = {}
|
|
524
|
+
existing_classes = {}
|
|
494
525
|
properties_by_class_by_subclass: list[_ReadProperties] = []
|
|
495
|
-
|
|
526
|
+
existing_class: InformationClass | None
|
|
527
|
+
total_instance_count = sum(count_by_type.values())
|
|
528
|
+
iterable = count_by_type.items()
|
|
529
|
+
if GLOBAL_CONFIG.use_iterate_bar_threshold and total_instance_count > GLOBAL_CONFIG.use_iterate_bar_threshold:
|
|
530
|
+
iterable = iterate_progress_bar(iterable, len(count_by_type), "Inferring types...") # type: ignore[assignment]
|
|
531
|
+
for type_uri, instance_count in iterable:
|
|
496
532
|
property_query = self._properties_query.format(type=type_uri, unknown_type=NEAT.UnknownType)
|
|
497
533
|
class_suffix = remove_namespace_from_uri(type_uri)
|
|
534
|
+
if (existing_class := existing_classes.get(class_suffix)) and existing_class.instance_source is None:
|
|
535
|
+
existing_class.instance_source = type_uri
|
|
536
|
+
|
|
498
537
|
for result_row in self.graph.query(property_query):
|
|
499
538
|
property_uri, value_type_uri = cast(tuple[URIRef, URIRef], result_row)
|
|
500
539
|
if property_uri == RDF.type:
|
|
501
540
|
continue
|
|
502
541
|
property_str = remove_namespace_from_uri(property_uri)
|
|
503
|
-
if (class_suffix, property_str)
|
|
542
|
+
if existing_property := existing_class_properties.get((class_suffix, property_str)):
|
|
543
|
+
if existing_property.instance_source is None:
|
|
544
|
+
existing_property.instance_source = [property_uri]
|
|
545
|
+
elif existing_property.instance_source and property_uri not in existing_property.instance_source:
|
|
546
|
+
existing_property.instance_source.append(property_uri)
|
|
504
547
|
continue
|
|
505
548
|
occurrence_query = self._max_occurrence_query.format(type=type_uri, property=property_uri)
|
|
506
549
|
max_occurrence = 1 # default value
|
|
@@ -531,21 +574,18 @@ class SubclassInferenceImporter(BaseRDFImporter):
|
|
|
531
574
|
self,
|
|
532
575
|
read_properties: list[_ReadProperties],
|
|
533
576
|
class_suffix: str,
|
|
534
|
-
type_uri: URIRef,
|
|
535
577
|
property_uri: URIRef,
|
|
578
|
+
property_id: str,
|
|
536
579
|
prefixes: dict[str, Namespace],
|
|
537
580
|
) -> InformationInputProperty:
|
|
538
581
|
first = read_properties[0]
|
|
539
582
|
value_type = self._get_value_type(read_properties, prefixes)
|
|
540
|
-
property_name = remove_namespace_from_uri(property_uri)
|
|
541
|
-
self._add_uri_namespace_to_prefixes(property_uri, prefixes)
|
|
542
|
-
|
|
543
583
|
return InformationInputProperty(
|
|
544
584
|
class_=class_suffix,
|
|
545
|
-
property_=
|
|
585
|
+
property_=property_id,
|
|
546
586
|
max_count=first.max_occurrence,
|
|
547
587
|
value_type=value_type,
|
|
548
|
-
instance_source=
|
|
588
|
+
instance_source=[property_uri],
|
|
549
589
|
)
|
|
550
590
|
|
|
551
591
|
def _get_value_type(
|
|
@@ -562,7 +602,7 @@ class SubclassInferenceImporter(BaseRDFImporter):
|
|
|
562
602
|
return UnknownEntity()
|
|
563
603
|
for uri_ref in value_types:
|
|
564
604
|
self._add_uri_namespace_to_prefixes(uri_ref, prefixes)
|
|
565
|
-
return "
|
|
605
|
+
return ", ".join(remove_namespace_from_uri(uri_ref) for uri_ref in value_types)
|
|
566
606
|
|
|
567
607
|
def _default_metadata(self) -> dict[str, Any]:
|
|
568
608
|
now = datetime.now(timezone.utc)
|
|
@@ -136,7 +136,7 @@ def parse_properties(graph: Graph, query: str, language: str, issue_list: IssueL
|
|
|
136
136
|
properties[id_]["value_type"].append(res["value_type"])
|
|
137
137
|
|
|
138
138
|
for prop in properties.values():
|
|
139
|
-
prop["value_type"] = "
|
|
139
|
+
prop["value_type"] = ", ".join(prop["value_type"])
|
|
140
140
|
|
|
141
141
|
if not properties:
|
|
142
142
|
issue_list.append(NeatValueError("Unable to parse properties"))
|
|
@@ -37,15 +37,15 @@ SOURCE_SHEET__TARGET_FIELD__HEADERS = [
|
|
|
37
37
|
"Properties",
|
|
38
38
|
"Properties",
|
|
39
39
|
{
|
|
40
|
-
RoleTypes.information: "Property",
|
|
41
|
-
RoleTypes.dms: "View Property",
|
|
40
|
+
RoleTypes.information: ["Class", "Property"],
|
|
41
|
+
RoleTypes.dms: ["View", "View Property"],
|
|
42
42
|
},
|
|
43
43
|
),
|
|
44
|
-
("Classes", "Classes", "Class"),
|
|
45
|
-
("Containers", "Containers", "Container"),
|
|
46
|
-
("Views", "Views", "View"),
|
|
47
|
-
("Enum", "Enum", "Collection"),
|
|
48
|
-
("Nodes", "Nodes", "Node"),
|
|
44
|
+
("Classes", "Classes", ["Class"]),
|
|
45
|
+
("Containers", "Containers", ["Container"]),
|
|
46
|
+
("Views", "Views", ["View"]),
|
|
47
|
+
("Enum", "Enum", ["Collection"]),
|
|
48
|
+
("Nodes", "Nodes", ["Node"]),
|
|
49
49
|
]
|
|
50
50
|
|
|
51
51
|
|
|
@@ -231,7 +231,10 @@ class SpreadsheetReader:
|
|
|
231
231
|
|
|
232
232
|
try:
|
|
233
233
|
sheets[target_sheet_name], read_info_by_sheet[source_sheet_name] = read_individual_sheet(
|
|
234
|
-
excel_file,
|
|
234
|
+
excel_file,
|
|
235
|
+
source_sheet_name,
|
|
236
|
+
return_read_info=True,
|
|
237
|
+
expected_headers=headers,
|
|
235
238
|
)
|
|
236
239
|
except Exception as e:
|
|
237
240
|
self.issue_list.append(FileReadError(cast(Path, excel_file.io), str(e)))
|
|
@@ -273,7 +276,7 @@ class ExcelImporter(BaseImporter[T_InputRules]):
|
|
|
273
276
|
|
|
274
277
|
rules_cls = INPUT_RULES_BY_ROLE[original_role]
|
|
275
278
|
rules = cast(T_InputRules, rules_cls.load(sheets))
|
|
276
|
-
return ReadRules(rules,
|
|
279
|
+
return ReadRules(rules, read_info_by_sheet)
|
|
277
280
|
|
|
278
281
|
@property
|
|
279
282
|
def description(self) -> str:
|
|
@@ -126,7 +126,6 @@ class SchemaModel(BaseModel):
|
|
|
126
126
|
extra="ignore",
|
|
127
127
|
use_enum_values=True,
|
|
128
128
|
)
|
|
129
|
-
validators_to_skip: set[str] = Field(default_factory=set, exclude=True)
|
|
130
129
|
|
|
131
130
|
@classmethod
|
|
132
131
|
def mandatory_fields(cls, use_alias=False) -> set[str]:
|
|
@@ -256,7 +255,6 @@ class BaseRules(SchemaModel, ABC):
|
|
|
256
255
|
|
|
257
256
|
Args:
|
|
258
257
|
metadata: Data model metadata
|
|
259
|
-
validators_to_skip: List of validators to skip. Defaults to []
|
|
260
258
|
"""
|
|
261
259
|
|
|
262
260
|
metadata: BaseMetadata
|
|
@@ -130,6 +130,13 @@ class DataType(BaseModel):
|
|
|
130
130
|
def as_xml_uri_ref(cls) -> URIRef:
|
|
131
131
|
return XML_SCHEMA_NAMESPACE[cls.xsd]
|
|
132
132
|
|
|
133
|
+
@classmethod
|
|
134
|
+
def convert_value(cls, value: Any) -> Any:
|
|
135
|
+
if cls != Boolean:
|
|
136
|
+
return cls.python(value)
|
|
137
|
+
else:
|
|
138
|
+
return value.strip().lower() in {"true", "1", "yes"}
|
|
139
|
+
|
|
133
140
|
|
|
134
141
|
class Boolean(DataType):
|
|
135
142
|
python = bool
|
|
@@ -99,6 +99,7 @@ class _DMSExporter:
|
|
|
99
99
|
key=lambda x: x.as_tuple(), # type: ignore[union-attr]
|
|
100
100
|
)
|
|
101
101
|
spaces = self._create_spaces(rules.metadata, containers, views, data_model)
|
|
102
|
+
|
|
102
103
|
return DMSSchema(
|
|
103
104
|
spaces=spaces,
|
|
104
105
|
data_model=data_model,
|
|
@@ -114,14 +115,14 @@ class _DMSExporter:
|
|
|
114
115
|
views: ViewApplyDict,
|
|
115
116
|
data_model: dm.DataModelApply,
|
|
116
117
|
) -> SpaceApplyDict:
|
|
117
|
-
used_spaces =
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
data_model.space
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
118
|
+
used_spaces = (
|
|
119
|
+
{container.space for container in containers.values()}
|
|
120
|
+
| {view.space for view in views.values()}
|
|
121
|
+
| {data_model.space}
|
|
122
|
+
| {metadata.space}
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
spaces = SpaceApplyDict([dm.SpaceApply(space=space) for space in used_spaces])
|
|
125
126
|
if self.instance_space and self.instance_space not in spaces:
|
|
126
127
|
spaces[self.instance_space] = dm.SpaceApply(space=self.instance_space, name=self.instance_space)
|
|
127
128
|
return spaces
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import warnings
|
|
1
2
|
from collections.abc import Hashable
|
|
2
3
|
from typing import TYPE_CHECKING, Any, ClassVar, Literal
|
|
3
4
|
|
|
@@ -8,6 +9,7 @@ from pydantic_core.core_schema import SerializationInfo, ValidationInfo
|
|
|
8
9
|
|
|
9
10
|
from cognite.neat._client.data_classes.schema import DMSSchema
|
|
10
11
|
from cognite.neat._issues.errors import NeatValueError
|
|
12
|
+
from cognite.neat._issues.warnings._general import NeatValueWarning
|
|
11
13
|
from cognite.neat._rules.models._base_rules import (
|
|
12
14
|
BaseMetadata,
|
|
13
15
|
BaseRules,
|
|
@@ -116,7 +118,7 @@ class DMSProperty(SheetRow):
|
|
|
116
118
|
description="Used to indicate whether the property holds single or multiple values (list). "
|
|
117
119
|
"Only applies to primitive types.",
|
|
118
120
|
)
|
|
119
|
-
default: str | int | dict | None = Field(
|
|
121
|
+
default: bool | str | int | float | dict | None = Field(
|
|
120
122
|
None, alias="Default", description="Specifies default value for the property."
|
|
121
123
|
)
|
|
122
124
|
container: ContainerEntityType | None = Field(
|
|
@@ -168,6 +170,29 @@ class DMSProperty(SheetRow):
|
|
|
168
170
|
raise ValueError(f"Reverse connection must have a value type that points to a view, got {value}")
|
|
169
171
|
return value
|
|
170
172
|
|
|
173
|
+
@field_validator("default", mode="after")
|
|
174
|
+
def set_proper_type_on_default(cls, value: Any, info: ValidationInfo) -> Any:
|
|
175
|
+
if not value:
|
|
176
|
+
return value
|
|
177
|
+
value_type = info.data.get("value_type")
|
|
178
|
+
if not isinstance(value_type, DataType):
|
|
179
|
+
warnings.filterwarnings("default")
|
|
180
|
+
warnings.warn(
|
|
181
|
+
NeatValueWarning(f"Default value {value} set to connection {value_type} will be ignored"),
|
|
182
|
+
stacklevel=2,
|
|
183
|
+
)
|
|
184
|
+
return None
|
|
185
|
+
else:
|
|
186
|
+
try:
|
|
187
|
+
return value_type.convert_value(value)
|
|
188
|
+
except ValueError:
|
|
189
|
+
warnings.filterwarnings("default")
|
|
190
|
+
warnings.warn(
|
|
191
|
+
NeatValueWarning(f"Could not convert {value} to {value_type}"),
|
|
192
|
+
stacklevel=2,
|
|
193
|
+
)
|
|
194
|
+
return None
|
|
195
|
+
|
|
171
196
|
@field_validator("container", "container_property", mode="after")
|
|
172
197
|
def container_set_correctly(cls, value: Any, info: ValidationInfo) -> Any:
|
|
173
198
|
if (connection := info.data.get("connection")) is None:
|
|
@@ -295,7 +320,9 @@ class DMSView(SheetRow):
|
|
|
295
320
|
None, alias="Filter", description="Explicitly define the filter for the view."
|
|
296
321
|
)
|
|
297
322
|
in_model: bool = Field(
|
|
298
|
-
True,
|
|
323
|
+
True,
|
|
324
|
+
alias="In Model",
|
|
325
|
+
description="Indicates whether the view being defined is a part of the data model.",
|
|
299
326
|
)
|
|
300
327
|
logical: URIRefType | None = Field(
|
|
301
328
|
None,
|
|
@@ -21,6 +21,7 @@ from cognite.neat._rules.models.entities import (
|
|
|
21
21
|
load_connection,
|
|
22
22
|
load_dms_value_type,
|
|
23
23
|
)
|
|
24
|
+
from cognite.neat._rules.models.entities._wrapped import DMSFilter
|
|
24
25
|
from cognite.neat._utils.rdf_ import uri_display_name
|
|
25
26
|
|
|
26
27
|
from ._rules import _DEFAULT_VERSION, DMSContainer, DMSEnum, DMSMetadata, DMSNode, DMSProperty, DMSRules, DMSView
|
|
@@ -69,7 +70,7 @@ class DMSInputMetadata(InputComponent[DMSMetadata]):
|
|
|
69
70
|
def _get_description_and_creator(cls, description_raw: str | None) -> tuple[str | None, list[str]]:
|
|
70
71
|
if description_raw and (description_match := re.search(r"Creator: (.+)", description_raw)):
|
|
71
72
|
creator = description_match.group(1).split(", ")
|
|
72
|
-
description = description_raw.replace(description_match
|
|
73
|
+
description = description_raw.replace(description_match[0], "").strip() or None
|
|
73
74
|
elif description_raw:
|
|
74
75
|
creator = ["MISSING"]
|
|
75
76
|
description = description_raw
|
|
@@ -193,6 +194,10 @@ class DMSInputView(InputComponent[DMSView]):
|
|
|
193
194
|
neatId: str | URIRef | None = None
|
|
194
195
|
logical: str | URIRef | None = None
|
|
195
196
|
|
|
197
|
+
def __post_init__(self):
|
|
198
|
+
if self.in_model is None:
|
|
199
|
+
self.in_model = True
|
|
200
|
+
|
|
196
201
|
@classmethod
|
|
197
202
|
def _get_verified_cls(cls) -> type[DMSView]:
|
|
198
203
|
return DMSView
|
|
@@ -207,6 +212,8 @@ class DMSInputView(InputComponent[DMSView]):
|
|
|
207
212
|
return ViewEntity.load(self.view, strict=True, space=default_space, version=default_version)
|
|
208
213
|
|
|
209
214
|
def _load_implements(self, default_space: str, default_version: str) -> list[ViewEntity] | None:
|
|
215
|
+
self.implements = self.implements.strip() if self.implements else None
|
|
216
|
+
|
|
210
217
|
return (
|
|
211
218
|
[
|
|
212
219
|
ViewEntity.load(implement, strict=True, space=default_space, version=default_version)
|
|
@@ -230,6 +237,7 @@ class DMSInputView(InputComponent[DMSView]):
|
|
|
230
237
|
implements=", ".join([str(ViewEntity.from_id(parent, _DEFAULT_VERSION)) for parent in view.implements])
|
|
231
238
|
or None,
|
|
232
239
|
in_model=in_model,
|
|
240
|
+
filter_=(str(DMSFilter.from_dms_filter(view.filter)) if view.filter else None),
|
|
233
241
|
)
|
|
234
242
|
|
|
235
243
|
|