cognite-neat 0.76.0__py3-none-any.whl → 0.76.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cognite/neat/_version.py +1 -1
- cognite/neat/app/api/routers/core.py +1 -1
- cognite/neat/app/api/routers/rules.py +1 -1
- cognite/neat/graph/extractors/_mock_graph_generator.py +2 -2
- cognite/neat/rules/_shared.py +1 -1
- cognite/neat/rules/analysis/_information_rules.py +3 -3
- cognite/neat/rules/exporters/_base.py +1 -1
- cognite/neat/rules/exporters/_rules2dms.py +8 -49
- cognite/neat/rules/exporters/_rules2excel.py +9 -3
- cognite/neat/rules/exporters/_rules2ontology.py +2 -2
- cognite/neat/rules/exporters/_rules2yaml.py +1 -1
- cognite/neat/rules/exporters/_validation.py +2 -2
- cognite/neat/rules/importers/_base.py +1 -1
- cognite/neat/rules/importers/_dms2rules.py +93 -108
- cognite/neat/rules/importers/_dtdl2rules/dtdl_converter.py +1 -1
- cognite/neat/rules/importers/_dtdl2rules/dtdl_importer.py +2 -3
- cognite/neat/rules/importers/_owl2rules/_owl2classes.py +1 -1
- cognite/neat/rules/importers/_owl2rules/_owl2metadata.py +2 -2
- cognite/neat/rules/importers/_owl2rules/_owl2properties.py +1 -1
- cognite/neat/rules/importers/_owl2rules/_owl2rules.py +1 -1
- cognite/neat/rules/importers/_spreadsheet2rules.py +10 -4
- cognite/neat/rules/importers/_yaml2rules.py +3 -3
- cognite/neat/rules/issues/base.py +5 -0
- cognite/neat/rules/models/__init__.py +27 -0
- cognite/neat/rules/models/dms/__init__.py +18 -0
- cognite/neat/rules/models/dms/_converter.py +140 -0
- cognite/neat/rules/models/dms/_exporter.py +405 -0
- cognite/neat/rules/models/dms/_rules.py +379 -0
- cognite/neat/rules/models/{rules/_dms_rules_write.py → dms/_rules_input.py} +65 -57
- cognite/neat/rules/models/{rules/_dms_schema.py → dms/_schema.py} +10 -4
- cognite/neat/rules/models/dms/_serializer.py +126 -0
- cognite/neat/rules/models/dms/_validation.py +255 -0
- cognite/neat/rules/models/information/__init__.py +3 -0
- cognite/neat/rules/models/information/_converter.py +193 -0
- cognite/neat/rules/models/{rules/_information_rules.py → information/_rules.py} +35 -202
- cognite/neat/workflows/steps/data_contracts.py +1 -1
- cognite/neat/workflows/steps/lib/current/rules_exporter.py +9 -3
- cognite/neat/workflows/steps/lib/current/rules_importer.py +1 -1
- cognite/neat/workflows/steps/lib/current/rules_validator.py +1 -2
- {cognite_neat-0.76.0.dist-info → cognite_neat-0.76.2.dist-info}/METADATA +1 -1
- {cognite_neat-0.76.0.dist-info → cognite_neat-0.76.2.dist-info}/RECORD +50 -43
- cognite/neat/rules/models/rules/__init__.py +0 -14
- cognite/neat/rules/models/rules/_dms_architect_rules.py +0 -1255
- /cognite/neat/rules/models/{rules/_base.py → _base.py} +0 -0
- /cognite/neat/rules/models/{rdfpath.py → _rdfpath.py} +0 -0
- /cognite/neat/rules/models/{rules/_types → _types}/__init__.py +0 -0
- /cognite/neat/rules/models/{rules/_types → _types}/_base.py +0 -0
- /cognite/neat/rules/models/{rules/_types → _types}/_field.py +0 -0
- /cognite/neat/rules/models/{rules/_domain_rules.py → domain.py} +0 -0
- {cognite_neat-0.76.0.dist-info → cognite_neat-0.76.2.dist-info}/LICENSE +0 -0
- {cognite_neat-0.76.0.dist-info → cognite_neat-0.76.2.dist-info}/WHEEL +0 -0
- {cognite_neat-0.76.0.dist-info → cognite_neat-0.76.2.dist-info}/entry_points.txt +0 -0
cognite/neat/_version.py
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
__version__ = "0.76.
|
|
1
|
+
__version__ = "0.76.2"
|
|
@@ -9,7 +9,7 @@ from fastapi import APIRouter, UploadFile
|
|
|
9
9
|
|
|
10
10
|
from cognite.neat.app.api.configuration import NEAT_APP
|
|
11
11
|
from cognite.neat.rules import exporters, importers
|
|
12
|
-
from cognite.neat.rules.models
|
|
12
|
+
from cognite.neat.rules.models import DMSRules, RoleTypes
|
|
13
13
|
|
|
14
14
|
router = APIRouter()
|
|
15
15
|
|
|
@@ -12,7 +12,7 @@ from cognite.neat.legacy.rules import importers as legacy_importers
|
|
|
12
12
|
from cognite.neat.legacy.rules.models._base import EntityTypes
|
|
13
13
|
from cognite.neat.legacy.rules.models.rules import Class, Classes, Metadata, Properties, Property, Rules
|
|
14
14
|
from cognite.neat.rules import importers
|
|
15
|
-
from cognite.neat.rules.models
|
|
15
|
+
from cognite.neat.rules.models import RoleTypes
|
|
16
16
|
from cognite.neat.workflows.steps.data_contracts import RulesData
|
|
17
17
|
from cognite.neat.workflows.steps.lib.current.rules_exporter import RulesToExcel
|
|
18
18
|
from cognite.neat.workflows.steps.lib.current.rules_importer import ExcelToRules
|
|
@@ -14,10 +14,10 @@ from rdflib import RDF, Literal, Namespace, URIRef
|
|
|
14
14
|
|
|
15
15
|
from cognite.neat.graph.models import Triple
|
|
16
16
|
from cognite.neat.rules.analysis import InformationArchitectRulesAnalysis
|
|
17
|
+
from cognite.neat.rules.models import DMSRules, InformationRules
|
|
17
18
|
from cognite.neat.rules.models.data_types import DataType
|
|
18
19
|
from cognite.neat.rules.models.entities import ClassEntity, EntityTypes
|
|
19
|
-
from cognite.neat.rules.models.
|
|
20
|
-
from cognite.neat.rules.models.rules._information_rules import InformationProperty
|
|
20
|
+
from cognite.neat.rules.models.information import InformationProperty
|
|
21
21
|
from cognite.neat.utils.utils import remove_namespace
|
|
22
22
|
|
|
23
23
|
from ._base import BaseExtractor
|
cognite/neat/rules/_shared.py
CHANGED
|
@@ -7,10 +7,10 @@ from typing import Any, cast
|
|
|
7
7
|
import pandas as pd
|
|
8
8
|
from pydantic import ValidationError
|
|
9
9
|
|
|
10
|
+
from cognite.neat.rules.models import SchemaCompleteness
|
|
11
|
+
from cognite.neat.rules.models._rdfpath import TransformationRuleType
|
|
10
12
|
from cognite.neat.rules.models.entities import ClassEntity, EntityTypes, ParentClassEntity, ReferenceEntity
|
|
11
|
-
from cognite.neat.rules.models.
|
|
12
|
-
from cognite.neat.rules.models.rules._base import SchemaCompleteness
|
|
13
|
-
from cognite.neat.rules.models.rules._information_rules import InformationClass, InformationProperty, InformationRules
|
|
13
|
+
from cognite.neat.rules.models.information import InformationClass, InformationProperty, InformationRules
|
|
14
14
|
from cognite.neat.utils.utils import get_inheritance_path
|
|
15
15
|
|
|
16
16
|
from ._base import BaseAnalysis, DataModelingScenario
|
|
@@ -6,7 +6,7 @@ from typing import Generic, TypeVar
|
|
|
6
6
|
from cognite.client import CogniteClient
|
|
7
7
|
|
|
8
8
|
from cognite.neat.rules._shared import Rules
|
|
9
|
-
from cognite.neat.rules.models
|
|
9
|
+
from cognite.neat.rules.models import DMSRules, InformationRules, RoleTypes
|
|
10
10
|
|
|
11
11
|
from ._models import UploadResult
|
|
12
12
|
|
|
@@ -11,10 +11,8 @@ from cognite.client.exceptions import CogniteAPIError
|
|
|
11
11
|
from cognite.neat.rules import issues
|
|
12
12
|
from cognite.neat.rules._shared import Rules
|
|
13
13
|
from cognite.neat.rules.issues import IssueList
|
|
14
|
-
from cognite.neat.rules.models
|
|
15
|
-
from cognite.neat.rules.models.
|
|
16
|
-
from cognite.neat.rules.models.rules._dms_architect_rules import DMSContainer, DMSRules
|
|
17
|
-
from cognite.neat.rules.models.rules._dms_schema import DMSSchema, PipelineSchema
|
|
14
|
+
from cognite.neat.rules.models import InformationRules
|
|
15
|
+
from cognite.neat.rules.models.dms import DMSRules, DMSSchema, PipelineSchema
|
|
18
16
|
from cognite.neat.utils.cdf_loaders import (
|
|
19
17
|
ContainerLoader,
|
|
20
18
|
DataModelingLoader,
|
|
@@ -116,47 +114,9 @@ class DMSExporter(CDFExporter[DMSSchema]):
|
|
|
116
114
|
dms_rules = rules.as_dms_architect_rules()
|
|
117
115
|
else:
|
|
118
116
|
raise ValueError(f"{type(rules).__name__} cannot be exported to DMS")
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
dms_rules.reference and dms_rules.metadata.external_id != dms_rules.reference.metadata.external_id
|
|
117
|
+
return dms_rules.as_schema(
|
|
118
|
+
include_ref=True, include_pipeline=self.export_pipeline, instance_space=self.instance_space
|
|
122
119
|
)
|
|
123
|
-
is_new_model = dms_rules.reference is None
|
|
124
|
-
if is_new_model or is_solution_model:
|
|
125
|
-
return dms_rules.as_schema(False, self.export_pipeline, self.instance_space)
|
|
126
|
-
|
|
127
|
-
# This is an extension of an existing model.
|
|
128
|
-
reference_rules = cast(DMSRules, dms_rules.reference).model_copy(deep=True)
|
|
129
|
-
reference_schema = reference_rules.as_schema(include_ref=False, include_pipeline=self.export_pipeline)
|
|
130
|
-
|
|
131
|
-
# Todo Move this to an appropriate location
|
|
132
|
-
# Merging Reference with User Rules
|
|
133
|
-
combined_rules = dms_rules.model_copy(deep=True)
|
|
134
|
-
existing_containers = {container.class_ for container in combined_rules.containers or []}
|
|
135
|
-
if combined_rules.containers is None:
|
|
136
|
-
combined_rules.containers = SheetList[DMSContainer](data=[])
|
|
137
|
-
for container in reference_rules.containers or []:
|
|
138
|
-
if container.class_ not in existing_containers:
|
|
139
|
-
container.reference = None
|
|
140
|
-
combined_rules.containers.append(container)
|
|
141
|
-
existing_views = {view.class_ for view in combined_rules.views}
|
|
142
|
-
for view in reference_rules.views:
|
|
143
|
-
if view.class_ not in existing_views:
|
|
144
|
-
view.reference = None
|
|
145
|
-
combined_rules.views.append(view)
|
|
146
|
-
existing_properties = {(property_.class_, property_.property_) for property_ in combined_rules.properties}
|
|
147
|
-
for property_ in reference_rules.properties:
|
|
148
|
-
if (property_.class_, property_.property_) not in existing_properties:
|
|
149
|
-
property_.reference = None
|
|
150
|
-
combined_rules.properties.append(property_)
|
|
151
|
-
|
|
152
|
-
schema = combined_rules.as_schema(True, self.export_pipeline, self.instance_space)
|
|
153
|
-
|
|
154
|
-
if dms_rules.metadata.extension in (ExtensionCategory.addition, ExtensionCategory.reshape):
|
|
155
|
-
# We do not freeze views as they might be changed, even for addition,
|
|
156
|
-
# in case, for example, new properties are added. The validation will catch this.
|
|
157
|
-
schema.frozen_ids.update(set(reference_schema.containers.as_ids()))
|
|
158
|
-
schema.frozen_ids.update(set(reference_schema.node_types.as_ids()))
|
|
159
|
-
return schema
|
|
160
120
|
|
|
161
121
|
def delete_from_cdf(self, rules: Rules, client: CogniteClient, dry_run: bool = False) -> Iterable[UploadResult]:
|
|
162
122
|
schema, to_export = self._prepare_schema_and_exporters(rules, client)
|
|
@@ -165,8 +125,7 @@ class DMSExporter(CDFExporter[DMSSchema]):
|
|
|
165
125
|
# as they are sorted in the order of creation and we need to delete them in reverse order
|
|
166
126
|
for all_items, loader in reversed(to_export):
|
|
167
127
|
all_item_ids = loader.get_ids(all_items)
|
|
168
|
-
|
|
169
|
-
item_ids = [item_id for item_id in all_item_ids if item_id not in schema.frozen_ids]
|
|
128
|
+
item_ids = [item_id for item_id in all_item_ids]
|
|
170
129
|
cdf_items = loader.retrieve(item_ids)
|
|
171
130
|
cdf_item_by_id = {loader.get_id(item): item for item in cdf_items}
|
|
172
131
|
items = [item for item in all_items if loader.get_id(item) in item_ids]
|
|
@@ -200,7 +159,7 @@ class DMSExporter(CDFExporter[DMSSchema]):
|
|
|
200
159
|
yield UploadResult(
|
|
201
160
|
name=loader.resource_name,
|
|
202
161
|
deleted=deleted,
|
|
203
|
-
skipped=
|
|
162
|
+
skipped=0,
|
|
204
163
|
failed_deleted=failed_deleted,
|
|
205
164
|
error_messages=error_messages,
|
|
206
165
|
)
|
|
@@ -215,8 +174,7 @@ class DMSExporter(CDFExporter[DMSSchema]):
|
|
|
215
174
|
for all_items, loader in to_export:
|
|
216
175
|
issue_list = IssueList()
|
|
217
176
|
all_item_ids = loader.get_ids(all_items)
|
|
218
|
-
|
|
219
|
-
item_ids = [item_id for item_id in all_item_ids if item_id not in schema.frozen_ids]
|
|
177
|
+
item_ids = [item_id for item_id in all_item_ids]
|
|
220
178
|
cdf_items = loader.retrieve(item_ids)
|
|
221
179
|
cdf_item_by_id = {loader.get_id(item): item for item in cdf_items}
|
|
222
180
|
items = [item for item in all_items if loader.get_id(item) in item_ids]
|
|
@@ -243,6 +201,7 @@ class DMSExporter(CDFExporter[DMSSchema]):
|
|
|
243
201
|
|
|
244
202
|
created = len(to_create)
|
|
245
203
|
failed_created = 0
|
|
204
|
+
skipped = 0
|
|
246
205
|
|
|
247
206
|
if self.existing_handling in ["update", "force"]:
|
|
248
207
|
changed = len(to_update)
|
|
@@ -12,8 +12,14 @@ from openpyxl.styles import Alignment, Border, Font, PatternFill, Side
|
|
|
12
12
|
from openpyxl.worksheet.worksheet import Worksheet
|
|
13
13
|
|
|
14
14
|
from cognite.neat.rules._shared import Rules
|
|
15
|
-
from cognite.neat.rules.models
|
|
16
|
-
|
|
15
|
+
from cognite.neat.rules.models import (
|
|
16
|
+
DMSRules,
|
|
17
|
+
DomainRules,
|
|
18
|
+
InformationRules,
|
|
19
|
+
RoleTypes,
|
|
20
|
+
SchemaCompleteness,
|
|
21
|
+
SheetEntity,
|
|
22
|
+
)
|
|
17
23
|
|
|
18
24
|
from ._base import BaseExporter
|
|
19
25
|
|
|
@@ -113,7 +119,7 @@ class ExcelExporter(BaseExporter[Workbook]):
|
|
|
113
119
|
|
|
114
120
|
def _write_sheets(self, workbook: Workbook, dumped_rules: dict[str, Any], rules: Rules, is_reference: bool = False):
|
|
115
121
|
for sheet_name, headers in rules.headers_by_sheet(by_alias=True).items():
|
|
116
|
-
if sheet_name in ("Metadata", "prefixes", "Reference", "
|
|
122
|
+
if sheet_name in ("Metadata", "prefixes", "Reference", "Last"):
|
|
117
123
|
continue
|
|
118
124
|
if is_reference:
|
|
119
125
|
sheet = workbook.create_sheet(f"Ref{sheet_name}")
|
|
@@ -11,10 +11,10 @@ from rdflib.collection import Collection as GraphCollection
|
|
|
11
11
|
from cognite.neat.constants import DEFAULT_NAMESPACE as NEAT_NAMESPACE
|
|
12
12
|
from cognite.neat.rules import exceptions
|
|
13
13
|
from cognite.neat.rules.analysis import InformationArchitectRulesAnalysis
|
|
14
|
+
from cognite.neat.rules.models import DMSRules
|
|
14
15
|
from cognite.neat.rules.models.data_types import DataType
|
|
15
16
|
from cognite.neat.rules.models.entities import ClassEntity, EntityTypes
|
|
16
|
-
from cognite.neat.rules.models.
|
|
17
|
-
from cognite.neat.rules.models.rules._information_rules import (
|
|
17
|
+
from cognite.neat.rules.models.information import (
|
|
18
18
|
InformationClass,
|
|
19
19
|
InformationMetadata,
|
|
20
20
|
InformationProperty,
|
|
@@ -4,8 +4,8 @@ from typing import Literal, overload
|
|
|
4
4
|
|
|
5
5
|
from cognite.neat.exceptions import wrangle_warnings
|
|
6
6
|
from cognite.neat.rules import exceptions
|
|
7
|
-
from cognite.neat.rules.models
|
|
8
|
-
from cognite.neat.rules.models.
|
|
7
|
+
from cognite.neat.rules.models import InformationRules
|
|
8
|
+
from cognite.neat.rules.models._types._base import DMS_PROPERTY_ID_COMPLIANCE_REGEX, VIEW_ID_COMPLIANCE_REGEX
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
@overload
|
|
@@ -11,7 +11,7 @@ from rdflib import Namespace
|
|
|
11
11
|
|
|
12
12
|
from cognite.neat.rules._shared import Rules
|
|
13
13
|
from cognite.neat.rules.issues.base import IssueList, NeatValidationError, ValidationWarning
|
|
14
|
-
from cognite.neat.rules.models
|
|
14
|
+
from cognite.neat.rules.models import DMSRules, InformationRules, RoleTypes
|
|
15
15
|
|
|
16
16
|
|
|
17
17
|
class BaseImporter(ABC):
|
|
@@ -2,7 +2,7 @@ from collections import Counter
|
|
|
2
2
|
from collections.abc import Sequence
|
|
3
3
|
from datetime import datetime
|
|
4
4
|
from pathlib import Path
|
|
5
|
-
from typing import Literal, cast, overload
|
|
5
|
+
from typing import Any, Literal, cast, overload
|
|
6
6
|
|
|
7
7
|
from cognite.client import CogniteClient
|
|
8
8
|
from cognite.client import data_modeling as dm
|
|
@@ -20,7 +20,22 @@ from cognite.client.utils import ms_to_datetime
|
|
|
20
20
|
from cognite.neat.rules import issues
|
|
21
21
|
from cognite.neat.rules.importers._base import BaseImporter, Rules, _handle_issues
|
|
22
22
|
from cognite.neat.rules.issues import IssueList, ValidationIssue
|
|
23
|
+
from cognite.neat.rules.models import (
|
|
24
|
+
DataModelType,
|
|
25
|
+
DMSRules,
|
|
26
|
+
DMSSchema,
|
|
27
|
+
ExtensionCategory,
|
|
28
|
+
RoleTypes,
|
|
29
|
+
SchemaCompleteness,
|
|
30
|
+
SheetList,
|
|
31
|
+
)
|
|
23
32
|
from cognite.neat.rules.models.data_types import DataType
|
|
33
|
+
from cognite.neat.rules.models.dms import (
|
|
34
|
+
DMSContainer,
|
|
35
|
+
DMSMetadata,
|
|
36
|
+
DMSProperty,
|
|
37
|
+
DMSView,
|
|
38
|
+
)
|
|
24
39
|
from cognite.neat.rules.models.entities import (
|
|
25
40
|
ClassEntity,
|
|
26
41
|
ContainerEntity,
|
|
@@ -29,15 +44,6 @@ from cognite.neat.rules.models.entities import (
|
|
|
29
44
|
ViewEntity,
|
|
30
45
|
ViewPropertyEntity,
|
|
31
46
|
)
|
|
32
|
-
from cognite.neat.rules.models.rules import DMSRules, DMSSchema, RoleTypes
|
|
33
|
-
from cognite.neat.rules.models.rules._base import DataModelType, ExtensionCategory, SchemaCompleteness
|
|
34
|
-
from cognite.neat.rules.models.rules._dms_architect_rules import (
|
|
35
|
-
DMSContainer,
|
|
36
|
-
DMSMetadata,
|
|
37
|
-
DMSProperty,
|
|
38
|
-
DMSView,
|
|
39
|
-
SheetList,
|
|
40
|
-
)
|
|
41
47
|
|
|
42
48
|
|
|
43
49
|
class DMSImporter(BaseImporter):
|
|
@@ -47,10 +53,17 @@ class DMSImporter(BaseImporter):
|
|
|
47
53
|
read_issues: Sequence[ValidationIssue] | None = None,
|
|
48
54
|
metadata: DMSMetadata | None = None,
|
|
49
55
|
):
|
|
50
|
-
|
|
56
|
+
# Calling this root schema to distinguish it from
|
|
57
|
+
# * User Schema
|
|
58
|
+
# * Reference Schema
|
|
59
|
+
self.root_schema = schema
|
|
51
60
|
self.metadata = metadata
|
|
52
61
|
self.issue_list = IssueList(read_issues)
|
|
53
|
-
self.
|
|
62
|
+
self._all_containers_by_id = {container.as_id(): container for container in schema.containers}
|
|
63
|
+
if self.root_schema.reference:
|
|
64
|
+
self._all_containers_by_id.update(
|
|
65
|
+
{container.as_id(): container for container in self.root_schema.reference.containers}
|
|
66
|
+
)
|
|
54
67
|
|
|
55
68
|
@classmethod
|
|
56
69
|
def from_data_model_id(cls, client: CogniteClient, data_model_id: DataModelIdentifier) -> "DMSImporter":
|
|
@@ -134,130 +147,102 @@ class DMSImporter(BaseImporter):
|
|
|
134
147
|
# In case there were errors during the import, the to_rules method will return None
|
|
135
148
|
return self._return_or_raise(self.issue_list, errors)
|
|
136
149
|
|
|
137
|
-
if len(self.
|
|
150
|
+
if len(self.root_schema.data_models) == 0:
|
|
138
151
|
self.issue_list.append(issues.importing.NoDataModelError("No data model found."))
|
|
139
152
|
return self._return_or_raise(self.issue_list, errors)
|
|
140
153
|
|
|
141
|
-
|
|
154
|
+
with _handle_issues(
|
|
155
|
+
self.issue_list,
|
|
156
|
+
) as future:
|
|
157
|
+
schema_completeness = SchemaCompleteness.complete
|
|
158
|
+
data_model_type = DataModelType.enterprise
|
|
159
|
+
reference: DMSRules | None = None
|
|
160
|
+
if ref_schema := self.root_schema.reference:
|
|
161
|
+
# Reference should always be an enterprise model.
|
|
162
|
+
reference = DMSRules(
|
|
163
|
+
**self._create_rule_components(
|
|
164
|
+
ref_schema, self._create_default_metadata(ref_schema.views), DataModelType.enterprise
|
|
165
|
+
)
|
|
166
|
+
)
|
|
167
|
+
schema_completeness = SchemaCompleteness.extended
|
|
168
|
+
data_model_type = DataModelType.solution
|
|
169
|
+
|
|
170
|
+
user_rules = DMSRules(
|
|
171
|
+
**self._create_rule_components(self.root_schema, self.metadata, data_model_type, schema_completeness),
|
|
172
|
+
reference=reference,
|
|
173
|
+
)
|
|
174
|
+
|
|
175
|
+
if future.result == "failure" or self.issue_list.has_errors:
|
|
176
|
+
return self._return_or_raise(self.issue_list, errors)
|
|
177
|
+
|
|
178
|
+
return self._to_output(user_rules, self.issue_list, errors, role)
|
|
179
|
+
|
|
180
|
+
def _create_rule_components(
|
|
181
|
+
self,
|
|
182
|
+
schema: DMSSchema,
|
|
183
|
+
metadata: DMSMetadata | None = None,
|
|
184
|
+
data_model_type: DataModelType | None = None,
|
|
185
|
+
schema_completeness: SchemaCompleteness | None = None,
|
|
186
|
+
) -> dict[str, Any]:
|
|
187
|
+
if len(schema.data_models) > 2:
|
|
142
188
|
# Creating a DataModelEntity to convert the data model id to a string.
|
|
143
189
|
self.issue_list.append(
|
|
144
190
|
issues.importing.MultipleDataModelsWarning(
|
|
145
|
-
[str(DataModelEntity.from_id(model.as_id())) for model in
|
|
191
|
+
[str(DataModelEntity.from_id(model.as_id())) for model in schema.data_models]
|
|
146
192
|
)
|
|
147
193
|
)
|
|
148
194
|
|
|
149
|
-
data_model =
|
|
195
|
+
data_model = schema.data_models[0]
|
|
150
196
|
|
|
151
197
|
properties = SheetList[DMSProperty]()
|
|
152
|
-
|
|
153
|
-
for view in self.schema.views:
|
|
198
|
+
for view in schema.views:
|
|
154
199
|
view_id = view.as_id()
|
|
155
200
|
view_entity = ViewEntity.from_id(view_id)
|
|
156
201
|
class_entity = view_entity.as_class()
|
|
157
202
|
for prop_id, prop in (view.properties or {}).items():
|
|
158
203
|
dms_property = self._create_dms_property(prop_id, prop, view_entity, class_entity)
|
|
159
204
|
if dms_property is not None:
|
|
160
|
-
|
|
161
|
-
ref_properties.append(dms_property)
|
|
162
|
-
else:
|
|
163
|
-
properties.append(dms_property)
|
|
205
|
+
properties.append(dms_property)
|
|
164
206
|
|
|
165
207
|
data_model_view_ids: set[dm.ViewId] = {
|
|
166
208
|
view.as_id() if isinstance(view, dm.View | dm.ViewApply) else view for view in data_model.views or []
|
|
167
209
|
}
|
|
168
210
|
|
|
169
|
-
metadata =
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
self.issue_list,
|
|
176
|
-
) as future:
|
|
177
|
-
user_rules = DMSRules(
|
|
178
|
-
metadata=metadata,
|
|
179
|
-
properties=properties,
|
|
180
|
-
containers=SheetList[DMSContainer](
|
|
181
|
-
data=[
|
|
182
|
-
DMSContainer.from_container(container)
|
|
183
|
-
for container in self.schema.containers
|
|
184
|
-
if container.as_id() not in self.schema.frozen_ids
|
|
185
|
-
]
|
|
186
|
-
),
|
|
187
|
-
views=SheetList[DMSView](
|
|
188
|
-
data=[
|
|
189
|
-
DMSView.from_view(view, in_model=view.as_id() in data_model_view_ids)
|
|
190
|
-
for view in self.schema.views
|
|
191
|
-
if view.as_id() not in self.schema.frozen_ids
|
|
192
|
-
]
|
|
193
|
-
),
|
|
194
|
-
reference=self._create_reference_rules(ref_properties),
|
|
195
|
-
)
|
|
196
|
-
|
|
197
|
-
if future.result == "failure" or self.issue_list.has_errors:
|
|
198
|
-
return self._return_or_raise(self.issue_list, errors)
|
|
199
|
-
|
|
200
|
-
return self._to_output(user_rules, self.issue_list, errors, role)
|
|
201
|
-
|
|
202
|
-
def _create_reference_rules(self, properties: SheetList[DMSProperty]) -> DMSRules | None:
|
|
203
|
-
if not properties:
|
|
204
|
-
return None
|
|
205
|
-
|
|
206
|
-
if len(self.schema.data_models) == 2:
|
|
207
|
-
data_model = self.schema.data_models[1]
|
|
208
|
-
data_model_view_ids: set[dm.ViewId] = {
|
|
209
|
-
view.as_id() if isinstance(view, dm.View | dm.ViewApply) else view for view in data_model.views or []
|
|
210
|
-
}
|
|
211
|
-
metadata = self._create_metadata_from_model(data_model)
|
|
212
|
-
else:
|
|
213
|
-
data_model_view_ids = set()
|
|
214
|
-
now = datetime.now().replace(microsecond=0)
|
|
215
|
-
space = Counter(prop.view.space for prop in properties).most_common(1)[0][0]
|
|
216
|
-
metadata = DMSMetadata(
|
|
217
|
-
schema_=SchemaCompleteness.complete,
|
|
218
|
-
extension=ExtensionCategory.addition,
|
|
219
|
-
space=space,
|
|
220
|
-
external_id="Unknown",
|
|
221
|
-
version="0.1.0",
|
|
222
|
-
creator=["Unknown"],
|
|
223
|
-
created=now,
|
|
224
|
-
updated=now,
|
|
225
|
-
)
|
|
226
|
-
|
|
227
|
-
metadata.data_model_type = DataModelType.enterprise
|
|
228
|
-
return DMSRules(
|
|
211
|
+
metadata = metadata or DMSMetadata.from_data_model(data_model)
|
|
212
|
+
if data_model_type is not None:
|
|
213
|
+
metadata.data_model_type = data_model_type
|
|
214
|
+
if schema_completeness is not None:
|
|
215
|
+
metadata.schema_ = schema_completeness
|
|
216
|
+
return dict(
|
|
229
217
|
metadata=metadata,
|
|
230
218
|
properties=properties,
|
|
231
|
-
views=SheetList[DMSView](
|
|
232
|
-
data=[
|
|
233
|
-
DMSView.from_view(view, in_model=not data_model_view_ids or (view.as_id() in data_model_view_ids))
|
|
234
|
-
for view in self.schema.views
|
|
235
|
-
if view.as_id() in self.schema.frozen_ids
|
|
236
|
-
]
|
|
237
|
-
),
|
|
238
219
|
containers=SheetList[DMSContainer](
|
|
239
|
-
data=[
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
]
|
|
220
|
+
data=[DMSContainer.from_container(container) for container in schema.containers]
|
|
221
|
+
),
|
|
222
|
+
views=SheetList[DMSView](
|
|
223
|
+
data=[DMSView.from_view(view, in_model=view.as_id() in data_model_view_ids) for view in schema.views]
|
|
244
224
|
),
|
|
245
|
-
reference=None,
|
|
246
225
|
)
|
|
247
226
|
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
227
|
+
@classmethod
|
|
228
|
+
def _create_default_metadata(cls, views: Sequence[dm.View | dm.ViewApply]) -> DMSMetadata:
|
|
229
|
+
now = datetime.now().replace(microsecond=0)
|
|
230
|
+
space = Counter(view.space for view in views).most_common(1)[0][0]
|
|
231
|
+
return DMSMetadata(
|
|
232
|
+
schema_=SchemaCompleteness.complete,
|
|
233
|
+
extension=ExtensionCategory.addition,
|
|
234
|
+
space=space,
|
|
235
|
+
external_id="Unknown",
|
|
236
|
+
version="0.1.0",
|
|
237
|
+
creator=["Unknown"],
|
|
238
|
+
created=now,
|
|
239
|
+
updated=now,
|
|
240
|
+
)
|
|
256
241
|
|
|
257
242
|
def _create_dms_property(
|
|
258
243
|
self, prop_id: str, prop: ViewPropertyApply, view_entity: ViewEntity, class_entity: ClassEntity
|
|
259
244
|
) -> DMSProperty | None:
|
|
260
|
-
if isinstance(prop, dm.MappedPropertyApply) and prop.container not in self.
|
|
245
|
+
if isinstance(prop, dm.MappedPropertyApply) and prop.container not in self._all_containers_by_id:
|
|
261
246
|
self.issue_list.append(
|
|
262
247
|
issues.importing.MissingContainerWarning(
|
|
263
248
|
view_id=str(view_entity),
|
|
@@ -268,7 +253,7 @@ class DMSImporter(BaseImporter):
|
|
|
268
253
|
return None
|
|
269
254
|
if (
|
|
270
255
|
isinstance(prop, dm.MappedPropertyApply)
|
|
271
|
-
and prop.container_property_identifier not in self.
|
|
256
|
+
and prop.container_property_identifier not in self._all_containers_by_id[prop.container].properties
|
|
272
257
|
):
|
|
273
258
|
self.issue_list.append(
|
|
274
259
|
issues.importing.MissingContainerPropertyWarning(
|
|
@@ -315,7 +300,7 @@ class DMSImporter(BaseImporter):
|
|
|
315
300
|
|
|
316
301
|
def _container_prop_unsafe(self, prop: dm.MappedPropertyApply) -> dm.ContainerProperty:
|
|
317
302
|
"""This method assumes you have already checked that the container with property exists."""
|
|
318
|
-
return self.
|
|
303
|
+
return self._all_containers_by_id[prop.container].properties[prop.container_property_identifier]
|
|
319
304
|
|
|
320
305
|
def _get_relation_type(self, prop: ViewPropertyApply) -> Literal["edge", "reverse", "direct"] | None:
|
|
321
306
|
if isinstance(prop, SingleEdgeConnectionApply | MultiEdgeConnectionApply) and prop.direction == "outwards":
|
|
@@ -380,7 +365,7 @@ class DMSImporter(BaseImporter):
|
|
|
380
365
|
def _get_index(self, prop: ViewPropertyApply, prop_id) -> list[str] | None:
|
|
381
366
|
if not isinstance(prop, dm.MappedPropertyApply):
|
|
382
367
|
return None
|
|
383
|
-
container = self.
|
|
368
|
+
container = self._all_containers_by_id[prop.container]
|
|
384
369
|
index: list[str] = []
|
|
385
370
|
for index_name, index_obj in (container.indexes or {}).items():
|
|
386
371
|
if isinstance(index_obj, BTreeIndex | InvertedIndex) and prop_id in index_obj.properties:
|
|
@@ -390,7 +375,7 @@ class DMSImporter(BaseImporter):
|
|
|
390
375
|
def _get_constraint(self, prop: ViewPropertyApply, prop_id: str) -> list[str] | None:
|
|
391
376
|
if not isinstance(prop, dm.MappedPropertyApply):
|
|
392
377
|
return None
|
|
393
|
-
container = self.
|
|
378
|
+
container = self._all_containers_by_id[prop.container]
|
|
394
379
|
unique_constraints: list[str] = []
|
|
395
380
|
for constraint_name, constraint_obj in (container.constraints or {}).items():
|
|
396
381
|
if isinstance(constraint_obj, dm.RequiresConstraint):
|
|
@@ -23,7 +23,7 @@ from cognite.neat.rules.importers._dtdl2rules.spec import (
|
|
|
23
23
|
from cognite.neat.rules.issues import IssueList, ValidationIssue
|
|
24
24
|
from cognite.neat.rules.models.data_types import _DATA_TYPE_BY_NAME, DataType, Json, String
|
|
25
25
|
from cognite.neat.rules.models.entities import ClassEntity, ParentClassEntity
|
|
26
|
-
from cognite.neat.rules.models.
|
|
26
|
+
from cognite.neat.rules.models.information import InformationClass, InformationProperty
|
|
27
27
|
|
|
28
28
|
|
|
29
29
|
class _DTDLConverter:
|
|
@@ -12,9 +12,8 @@ from cognite.neat.rules.importers._base import BaseImporter, _handle_issues
|
|
|
12
12
|
from cognite.neat.rules.importers._dtdl2rules.dtdl_converter import _DTDLConverter
|
|
13
13
|
from cognite.neat.rules.importers._dtdl2rules.spec import DTDL_CLS_BY_TYPE_BY_SPEC, DTDLBase, Interface
|
|
14
14
|
from cognite.neat.rules.issues import IssueList, ValidationIssue
|
|
15
|
-
from cognite.neat.rules.models
|
|
16
|
-
from cognite.neat.rules.models.
|
|
17
|
-
from cognite.neat.rules.models.rules._information_rules import InformationClass, InformationProperty
|
|
15
|
+
from cognite.neat.rules.models import InformationRules, RoleTypes, SchemaCompleteness, SheetList
|
|
16
|
+
from cognite.neat.rules.models.information import InformationClass, InformationProperty
|
|
18
17
|
from cognite.neat.utils.text import to_pascal
|
|
19
18
|
|
|
20
19
|
|
|
@@ -4,8 +4,8 @@ import re
|
|
|
4
4
|
from rdflib import Graph, Namespace
|
|
5
5
|
|
|
6
6
|
from cognite.neat.constants import DEFAULT_NAMESPACE
|
|
7
|
-
from cognite.neat.rules.models
|
|
8
|
-
from cognite.neat.rules.models.
|
|
7
|
+
from cognite.neat.rules.models import RoleTypes, SchemaCompleteness
|
|
8
|
+
from cognite.neat.rules.models._types._base import (
|
|
9
9
|
PREFIX_COMPLIANCE_REGEX,
|
|
10
10
|
VERSION_COMPLIANCE_REGEX,
|
|
11
11
|
)
|
|
@@ -4,7 +4,7 @@ import numpy as np
|
|
|
4
4
|
import pandas as pd
|
|
5
5
|
from rdflib import Graph
|
|
6
6
|
|
|
7
|
-
from cognite.neat.rules.models.
|
|
7
|
+
from cognite.neat.rules.models._base import MatchType
|
|
8
8
|
from cognite.neat.utils.utils import remove_namespace
|
|
9
9
|
|
|
10
10
|
from ._owl2classes import _data_type_property_class, _object_property_class, _thing_class
|
|
@@ -10,8 +10,8 @@ from rdflib import DC, DCTERMS, OWL, RDF, RDFS, SKOS, Graph
|
|
|
10
10
|
|
|
11
11
|
from cognite.neat.rules.importers._base import BaseImporter, Rules
|
|
12
12
|
from cognite.neat.rules.issues import IssueList
|
|
13
|
+
from cognite.neat.rules.models import InformationRules, RoleTypes
|
|
13
14
|
from cognite.neat.rules.models.data_types import _XSD_TYPES
|
|
14
|
-
from cognite.neat.rules.models.rules import InformationRules, RoleTypes
|
|
15
15
|
|
|
16
16
|
from ._owl2classes import parse_owl_classes
|
|
17
17
|
from ._owl2metadata import parse_owl_metadata
|
|
@@ -13,9 +13,15 @@ from pandas import ExcelFile
|
|
|
13
13
|
|
|
14
14
|
from cognite.neat.rules import issues
|
|
15
15
|
from cognite.neat.rules.issues import IssueList
|
|
16
|
-
from cognite.neat.rules.models
|
|
17
|
-
|
|
18
|
-
|
|
16
|
+
from cognite.neat.rules.models import (
|
|
17
|
+
RULES_PER_ROLE,
|
|
18
|
+
DMSRules,
|
|
19
|
+
DomainRules,
|
|
20
|
+
InformationRules,
|
|
21
|
+
RoleTypes,
|
|
22
|
+
SchemaCompleteness,
|
|
23
|
+
)
|
|
24
|
+
from cognite.neat.rules.models.dms import DMSRulesInput
|
|
19
25
|
from cognite.neat.utils.auxiliary import local_import
|
|
20
26
|
from cognite.neat.utils.spreadsheet import SpreadsheetRead, read_individual_sheet
|
|
21
27
|
|
|
@@ -227,7 +233,7 @@ class ExcelImporter(BaseImporter):
|
|
|
227
233
|
) as future:
|
|
228
234
|
rules: Rules
|
|
229
235
|
if rules_cls is DMSRules:
|
|
230
|
-
rules =
|
|
236
|
+
rules = DMSRulesInput.load(sheets).as_rules()
|
|
231
237
|
else:
|
|
232
238
|
rules = rules_cls.model_validate(sheets) # type: ignore[attr-defined]
|
|
233
239
|
|
|
@@ -5,8 +5,8 @@ import yaml
|
|
|
5
5
|
|
|
6
6
|
from cognite.neat.rules import issues
|
|
7
7
|
from cognite.neat.rules.issues import IssueList, NeatValidationError, ValidationIssue
|
|
8
|
-
from cognite.neat.rules.models
|
|
9
|
-
from cognite.neat.rules.models.
|
|
8
|
+
from cognite.neat.rules.models import RULES_PER_ROLE, DMSRules, RoleTypes
|
|
9
|
+
from cognite.neat.rules.models.dms import DMSRulesInput
|
|
10
10
|
|
|
11
11
|
from ._base import BaseImporter, Rules, _handle_issues
|
|
12
12
|
|
|
@@ -98,7 +98,7 @@ class YAMLImporter(BaseImporter):
|
|
|
98
98
|
with _handle_issues(issue_list) as future:
|
|
99
99
|
rules: Rules
|
|
100
100
|
if rules_model is DMSRules:
|
|
101
|
-
rules =
|
|
101
|
+
rules = DMSRulesInput.load(self.raw_data).as_rules()
|
|
102
102
|
else:
|
|
103
103
|
rules = rules_model.model_validate(self.raw_data)
|
|
104
104
|
|