cognite-neat 0.72.2__py3-none-any.whl → 0.73.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_version.py +1 -1
- cognite/neat/constants.py +3 -2
- cognite/neat/graph/extractor/_graph_capturing_sheet.py +14 -12
- cognite/neat/rules/examples/power-grid-containers.yaml +3 -0
- cognite/neat/rules/examples/power-grid-model.yaml +3 -0
- cognite/neat/rules/exporter/_rules2dms.py +3 -10
- cognite/neat/rules/exporter/_rules2excel.py +3 -2
- cognite/neat/rules/exporters/_rules2dms.py +3 -7
- cognite/neat/rules/exporters/_rules2excel.py +94 -4
- cognite/neat/rules/exporters/_rules2ontology.py +2 -0
- cognite/neat/rules/importer/_dms2rules.py +1 -4
- cognite/neat/rules/importers/_dms2rules.py +40 -11
- cognite/neat/rules/importers/_owl2rules/_owl2metadata.py +15 -15
- cognite/neat/rules/importers/_owl2rules/_owl2properties.py +2 -0
- cognite/neat/rules/importers/_owl2rules/_owl2rules.py +1 -1
- cognite/neat/rules/importers/_spreadsheet2rules.py +52 -31
- cognite/neat/rules/issues/base.py +9 -1
- cognite/neat/rules/issues/dms.py +74 -0
- cognite/neat/rules/models/_rules/_types/_base.py +6 -16
- cognite/neat/rules/models/_rules/base.py +24 -2
- cognite/neat/rules/models/_rules/dms_architect_rules.py +181 -71
- cognite/neat/rules/models/_rules/dms_schema.py +5 -1
- cognite/neat/rules/models/_rules/domain_rules.py +14 -1
- cognite/neat/rules/models/_rules/information_rules.py +16 -2
- cognite/neat/utils/spreadsheet.py +2 -2
- cognite/neat/workflows/steps/lib/rules_exporter.py +0 -1
- {cognite_neat-0.72.2.dist-info → cognite_neat-0.73.0.dist-info}/METADATA +2 -2
- {cognite_neat-0.72.2.dist-info → cognite_neat-0.73.0.dist-info}/RECORD +31 -32
- cognite/neat/py.typed +0 -0
- {cognite_neat-0.72.2.dist-info → cognite_neat-0.73.0.dist-info}/LICENSE +0 -0
- {cognite_neat-0.72.2.dist-info → cognite_neat-0.73.0.dist-info}/WHEEL +0 -0
- {cognite_neat-0.72.2.dist-info → cognite_neat-0.73.0.dist-info}/entry_points.txt +0 -0
cognite/neat/_version.py
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
__version__ = "0.
|
|
1
|
+
__version__ = "0.73.0"
|
cognite/neat/constants.py
CHANGED
|
@@ -11,6 +11,7 @@ EXAMPLE_RULES = PACKAGE_DIRECTORY / "rules" / "examples"
|
|
|
11
11
|
EXAMPLE_GRAPHS = PACKAGE_DIRECTORY / "graph" / "examples"
|
|
12
12
|
EXAMPLE_WORKFLOWS = PACKAGE_DIRECTORY / "workflows" / "examples"
|
|
13
13
|
|
|
14
|
+
DEFAULT_NAMESPACE = Namespace("http://purl.org/cognite/neat#")
|
|
14
15
|
|
|
15
16
|
PREFIXES = {
|
|
16
17
|
"rdf": RDF._NS,
|
|
@@ -27,10 +28,10 @@ PREFIXES = {
|
|
|
27
28
|
"md": Namespace("http://iec.ch/TC57/61970-552/ModelDescription/1#"),
|
|
28
29
|
"pti": Namespace("http://www.pti-us.com/PTI_CIM-schema-cim16#"),
|
|
29
30
|
"tnt": Namespace("http://purl.org/cognite/tnt#"),
|
|
30
|
-
"neat":
|
|
31
|
+
"neat": DEFAULT_NAMESPACE,
|
|
31
32
|
}
|
|
32
33
|
|
|
33
|
-
|
|
34
|
+
|
|
34
35
|
DEFAULT_URI = ""
|
|
35
36
|
|
|
36
37
|
DEFAULT_DOCS_URL = "https://cognite-neat.readthedocs-hosted.com/en/latest/"
|
|
@@ -11,6 +11,7 @@ from openpyxl.cell import Cell
|
|
|
11
11
|
from openpyxl.styles import Alignment, Border, Font, NamedStyle, PatternFill, Side
|
|
12
12
|
from openpyxl.utils import get_column_letter
|
|
13
13
|
from openpyxl.worksheet.datavalidation import DataValidation
|
|
14
|
+
from openpyxl.worksheet.worksheet import Worksheet
|
|
14
15
|
from rdflib import RDF, XSD, Literal, Namespace, URIRef
|
|
15
16
|
|
|
16
17
|
from cognite.neat.graph import exceptions
|
|
@@ -321,7 +322,7 @@ def rules2graph_capturing_sheet(
|
|
|
321
322
|
workbook.create_sheet(title=class_)
|
|
322
323
|
|
|
323
324
|
# Add header rows
|
|
324
|
-
workbook[class_].append(["identifier", *list(properties.keys())])
|
|
325
|
+
cast(Worksheet, workbook[class_]).append(["identifier", *list(properties.keys())])
|
|
325
326
|
|
|
326
327
|
if auto_identifier_type and auto_identifier_type == "index-based": # default, easy to read
|
|
327
328
|
logging.debug(f"Configuring index-based automatic identifiers for sheet {class_}")
|
|
@@ -350,35 +351,35 @@ def _add_index_identifiers(workbook: Workbook, sheet: str, no_rows: int):
|
|
|
350
351
|
"""Adds index-based auto identifier to a sheet identifier column"""
|
|
351
352
|
for i in range(no_rows):
|
|
352
353
|
prefix = to_dms_name(sheet, "class", True)
|
|
353
|
-
workbook[sheet][f"A{i+2}"] = f'=IF(ISBLANK(B{i+2}), "","{prefix}-{i+1}")'
|
|
354
|
+
workbook[sheet][f"A{i+2}"] = f'=IF(ISBLANK(B{i+2}), "","{prefix}-{i+1}")' # type: ignore[index]
|
|
354
355
|
|
|
355
356
|
|
|
356
357
|
def _add_uuid_identifiers(workbook: Workbook, sheet: str, no_rows: int):
|
|
357
358
|
"""Adds UUID-based auto identifier to a sheet identifier column"""
|
|
358
359
|
for i in range(no_rows):
|
|
359
360
|
prefix = to_dms_name(sheet, "class", True)
|
|
360
|
-
workbook[sheet][f"A{i+2}"] = f'=IF(ISBLANK(B{i+2}), "","{prefix}-{uuid.uuid4()}")'
|
|
361
|
+
workbook[sheet][f"A{i+2}"] = f'=IF(ISBLANK(B{i+2}), "","{prefix}-{uuid.uuid4()}")' # type: ignore[index]
|
|
361
362
|
|
|
362
363
|
|
|
363
364
|
def _add_drop_down_list(workbook: Workbook, sheet: str, column: str, no_rows: int, value_sheet: str, value_column: str):
|
|
364
365
|
"""Adds a drop down list to a column"""
|
|
365
366
|
drop_down_list = DataValidation(type="list", formula1=f"={value_sheet}!{value_column}$2:{value_column}${no_rows}")
|
|
366
367
|
|
|
367
|
-
workbook[sheet].add_data_validation(drop_down_list)
|
|
368
|
+
cast(Worksheet, workbook[sheet]).add_data_validation(drop_down_list)
|
|
368
369
|
|
|
369
370
|
for i in range(no_rows):
|
|
370
|
-
drop_down_list.add(workbook[sheet][f"{column}{i+2}"])
|
|
371
|
+
drop_down_list.add(workbook[sheet][f"{column}{i+2}"]) # type: ignore[index, misc]
|
|
371
372
|
|
|
372
373
|
|
|
373
374
|
def _adjust_column_width(workbook: Workbook):
|
|
374
375
|
"""Adjusts the column width based on the content"""
|
|
375
376
|
for sheet in workbook.sheetnames:
|
|
376
|
-
for cell_tuple in workbook[sheet].columns:
|
|
377
|
+
for cell_tuple in cast(Worksheet, workbook[sheet]).columns:
|
|
377
378
|
# Wrong type annotation in openpyxl
|
|
378
379
|
cell = cast(Cell, cell_tuple[0]) # type: ignore[index]
|
|
379
380
|
if cell.value:
|
|
380
381
|
adjusted_width = (len(str(cell.value)) + 5) * 1.2
|
|
381
|
-
workbook[sheet].column_dimensions[cell.column_letter].width = adjusted_width
|
|
382
|
+
cast(Worksheet, workbook[sheet]).column_dimensions[cell.column_letter].width = adjusted_width
|
|
382
383
|
|
|
383
384
|
|
|
384
385
|
def _set_header_style(workbook: Workbook):
|
|
@@ -390,12 +391,13 @@ def _set_header_style(workbook: Workbook):
|
|
|
390
391
|
workbook.add_named_style(style)
|
|
391
392
|
|
|
392
393
|
for sheet in workbook.sheetnames:
|
|
393
|
-
for cell_tuple in workbook[sheet].columns:
|
|
394
|
+
for cell_tuple in cast(Worksheet, workbook[sheet]).columns:
|
|
394
395
|
# Wrong type annotation in openpyxl
|
|
395
396
|
cell = cast(Cell, cell_tuple[0]) # type: ignore[index]
|
|
396
|
-
workbook[sheet]
|
|
397
|
+
worksheet = cast(Worksheet, workbook[sheet])
|
|
398
|
+
worksheet[f"{cell.column_letter}1"].style = style
|
|
397
399
|
if f"{cell.column_letter}1" == "A1":
|
|
398
|
-
|
|
400
|
+
worksheet[f"{cell.column_letter}1"].fill = PatternFill("solid", start_color="2FB5F2")
|
|
399
401
|
else:
|
|
400
|
-
|
|
401
|
-
|
|
402
|
+
worksheet[f"{cell.column_letter}1"].fill = PatternFill("solid", start_color="FFB202")
|
|
403
|
+
worksheet[f"{cell.column_letter}1"].alignment = Alignment(horizontal="center", vertical="center")
|
|
@@ -53,6 +53,7 @@
|
|
|
53
53
|
type:
|
|
54
54
|
container: null
|
|
55
55
|
type: direct
|
|
56
|
+
list: false
|
|
56
57
|
space: workshop
|
|
57
58
|
usedFor: node
|
|
58
59
|
- externalId: SubGeographicalRegion
|
|
@@ -81,6 +82,7 @@
|
|
|
81
82
|
type:
|
|
82
83
|
container: null
|
|
83
84
|
type: direct
|
|
85
|
+
list: false
|
|
84
86
|
space: workshop
|
|
85
87
|
usedFor: node
|
|
86
88
|
- externalId: Substation
|
|
@@ -109,5 +111,6 @@
|
|
|
109
111
|
type:
|
|
110
112
|
container: null
|
|
111
113
|
type: direct
|
|
114
|
+
list: false
|
|
112
115
|
space: workshop
|
|
113
116
|
usedFor: node
|
|
@@ -81,6 +81,7 @@ views:
|
|
|
81
81
|
type:
|
|
82
82
|
container: null
|
|
83
83
|
type: direct
|
|
84
|
+
list: false
|
|
84
85
|
source:
|
|
85
86
|
space: workshop
|
|
86
87
|
externalId: GeographicalRegion
|
|
@@ -132,6 +133,7 @@ views:
|
|
|
132
133
|
type:
|
|
133
134
|
container: null
|
|
134
135
|
type: direct
|
|
136
|
+
list: false
|
|
135
137
|
source:
|
|
136
138
|
space: workshop
|
|
137
139
|
externalId: SubGeographicalRegion
|
|
@@ -195,6 +197,7 @@ views:
|
|
|
195
197
|
type:
|
|
196
198
|
container: null
|
|
197
199
|
type: direct
|
|
200
|
+
list: false
|
|
198
201
|
source:
|
|
199
202
|
space: workshop
|
|
200
203
|
externalId: Substation
|
|
@@ -34,7 +34,6 @@ from cognite.client.data_classes.data_modeling import (
|
|
|
34
34
|
ViewApply,
|
|
35
35
|
ViewId,
|
|
36
36
|
)
|
|
37
|
-
from cognite.client.data_classes.data_modeling.data_types import ListablePropertyType
|
|
38
37
|
from cognite.client.data_classes.data_modeling.views import (
|
|
39
38
|
ConnectionDefinitionApply,
|
|
40
39
|
SingleHopConnectionDefinitionApply,
|
|
@@ -284,12 +283,9 @@ class DMSSchemaComponents(BaseModel):
|
|
|
284
283
|
if (
|
|
285
284
|
not isinstance(existing_property.type, DirectRelation)
|
|
286
285
|
and not isinstance(api_container_property.type, DirectRelation)
|
|
287
|
-
and
|
|
288
|
-
!= cast(ListablePropertyType, api_container_property.type).is_list
|
|
286
|
+
and existing_property.type.is_list != api_container_property.type.is_list
|
|
289
287
|
):
|
|
290
|
-
|
|
291
|
-
ListablePropertyType, containers[container_id].properties[container_property_id].type
|
|
292
|
-
).is_list = True
|
|
288
|
+
containers[container_id].properties[container_property_id].type.is_list = True
|
|
293
289
|
|
|
294
290
|
if errors:
|
|
295
291
|
raise ExceptionGroup("Properties value types have been redefined! This is prohibited! Aborting!", errors)
|
|
@@ -302,10 +298,7 @@ class DMSSchemaComponents(BaseModel):
|
|
|
302
298
|
|
|
303
299
|
# Literal, i.e. Node attribute
|
|
304
300
|
if property_.property_type is EntityTypes.data_property:
|
|
305
|
-
property_type = cast(
|
|
306
|
-
type[ListablePropertyType],
|
|
307
|
-
cast(ValueTypeMapping, property_.expected_value_type.mapping).dms,
|
|
308
|
-
)
|
|
301
|
+
property_type = cast(ValueTypeMapping, property_.expected_value_type.mapping).dms
|
|
309
302
|
return ContainerProperty(
|
|
310
303
|
type=property_type(is_list=is_one_to_many),
|
|
311
304
|
nullable=property_.min_count == 0,
|
|
@@ -4,6 +4,7 @@ from typing import cast
|
|
|
4
4
|
from openpyxl import Workbook
|
|
5
5
|
from openpyxl.cell import Cell
|
|
6
6
|
from openpyxl.styles import Alignment, Border, Font, NamedStyle, PatternFill, Side
|
|
7
|
+
from openpyxl.worksheet.worksheet import Worksheet
|
|
7
8
|
|
|
8
9
|
from cognite.neat.rules.models._base import EntityTypes
|
|
9
10
|
|
|
@@ -190,7 +191,7 @@ class ExcelExporter(BaseExporter[Workbook]):
|
|
|
190
191
|
if sheet == "Metadata":
|
|
191
192
|
continue
|
|
192
193
|
if sheet == "Classes" or sheet == "Properties":
|
|
193
|
-
sheet_obj = data[sheet]
|
|
194
|
+
sheet_obj = cast(Worksheet, data[sheet])
|
|
194
195
|
if sheet == "Classes":
|
|
195
196
|
sheet_obj.freeze_panes = "A3"
|
|
196
197
|
else:
|
|
@@ -207,6 +208,6 @@ class ExcelExporter(BaseExporter[Workbook]):
|
|
|
207
208
|
cell.fill = PatternFill("solid", start_color="D5DBD5")
|
|
208
209
|
cell.alignment = Alignment(horizontal="center", vertical="center")
|
|
209
210
|
adjusted_width = (len(str(cell.value)) + 5) * 1.2
|
|
210
|
-
data[sheet].column_dimensions[cell.column_letter].width = adjusted_width
|
|
211
|
+
cast(Worksheet, data[sheet]).column_dimensions[cell.column_letter].width = adjusted_width
|
|
211
212
|
|
|
212
213
|
return data
|
|
@@ -40,8 +40,6 @@ class DMSExporter(CDFExporter[DMSSchema]):
|
|
|
40
40
|
If set, only export components in the given spaces. Defaults to None which means all spaces.
|
|
41
41
|
existing_handling (Literal["fail", "skip", "update", "force"], optional): How to handle existing components.
|
|
42
42
|
Defaults to "update". See below for details.
|
|
43
|
-
standardize_casing(bool, optional): Whether to standardize the casing. This means PascalCase for external ID
|
|
44
|
-
of views, containers, and data models, and camelCase for properties.
|
|
45
43
|
export_pipeline (bool, optional): Whether to export the pipeline. Defaults to False. This means setting
|
|
46
44
|
up transformations, RAW databases and tables to populate the data model.
|
|
47
45
|
instance_space (str, optional): The space to use for the instance. Defaults to None.
|
|
@@ -59,14 +57,12 @@ class DMSExporter(CDFExporter[DMSSchema]):
|
|
|
59
57
|
export_components: Component | Collection[Component] = "all",
|
|
60
58
|
include_space: set[str] | None = None,
|
|
61
59
|
existing_handling: Literal["fail", "skip", "update", "force"] = "update",
|
|
62
|
-
standardize_casing: bool = True,
|
|
63
60
|
export_pipeline: bool = False,
|
|
64
61
|
instance_space: str | None = None,
|
|
65
62
|
):
|
|
66
63
|
self.export_components = {export_components} if isinstance(export_components, str) else set(export_components)
|
|
67
64
|
self.include_space = include_space
|
|
68
65
|
self.existing_handling = existing_handling
|
|
69
|
-
self.standardize_casing = standardize_casing
|
|
70
66
|
self.export_pipeline = export_pipeline
|
|
71
67
|
self.instance_space = instance_space
|
|
72
68
|
self._schema: DMSSchema | None = None
|
|
@@ -119,11 +115,11 @@ class DMSExporter(CDFExporter[DMSSchema]):
|
|
|
119
115
|
)
|
|
120
116
|
is_new_model = dms_rules.reference is None
|
|
121
117
|
if is_new_model or is_solution_model:
|
|
122
|
-
return dms_rules.as_schema(self.
|
|
118
|
+
return dms_rules.as_schema(self.export_pipeline, self.instance_space)
|
|
123
119
|
|
|
124
120
|
# This is an extension of an existing model.
|
|
125
121
|
reference_rules = cast(DMSRules, dms_rules.reference).copy(deep=True)
|
|
126
|
-
reference_schema = reference_rules.as_schema(self.
|
|
122
|
+
reference_schema = reference_rules.as_schema(self.export_pipeline)
|
|
127
123
|
|
|
128
124
|
# Todo Move this to an appropriate location
|
|
129
125
|
# Merging Reference with User Rules
|
|
@@ -146,7 +142,7 @@ class DMSExporter(CDFExporter[DMSSchema]):
|
|
|
146
142
|
property_.reference = None
|
|
147
143
|
combined_rules.properties.append(property_)
|
|
148
144
|
|
|
149
|
-
schema = combined_rules.as_schema(self.
|
|
145
|
+
schema = combined_rules.as_schema(self.export_pipeline, self.instance_space)
|
|
150
146
|
|
|
151
147
|
if dms_rules.metadata.extension in (ExtensionCategory.addition, ExtensionCategory.reshape):
|
|
152
148
|
# We do not freeze views as they might be changed, even for addition,
|
|
@@ -1,16 +1,19 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import itertools
|
|
4
|
+
from datetime import datetime
|
|
4
5
|
from pathlib import Path
|
|
5
6
|
from types import GenericAlias
|
|
6
|
-
from typing import Any, ClassVar, Literal, get_args
|
|
7
|
+
from typing import Any, ClassVar, Literal, cast, get_args
|
|
7
8
|
|
|
8
9
|
from openpyxl import Workbook
|
|
9
10
|
from openpyxl.cell import MergedCell
|
|
10
11
|
from openpyxl.styles import Alignment, Border, Font, PatternFill, Side
|
|
12
|
+
from openpyxl.worksheet.worksheet import Worksheet
|
|
11
13
|
|
|
12
14
|
from cognite.neat.rules._shared import Rules
|
|
13
|
-
from cognite.neat.rules.models._rules
|
|
15
|
+
from cognite.neat.rules.models._rules import DMSRules, DomainRules, InformationRules
|
|
16
|
+
from cognite.neat.rules.models._rules.base import RoleTypes, SchemaCompleteness, SheetEntity
|
|
14
17
|
|
|
15
18
|
from ._base import BaseExporter
|
|
16
19
|
|
|
@@ -23,6 +26,9 @@ class ExcelExporter(BaseExporter[Workbook]):
|
|
|
23
26
|
on the different styles.
|
|
24
27
|
output_role: The role to use for the exported spreadsheet. If provided, the rules will be converted to
|
|
25
28
|
this role formate before being written to excel. If not provided, the role from the rules will be used.
|
|
29
|
+
new_model_id: The new model ID to use for the exported spreadsheet. This is only applicable if the input
|
|
30
|
+
rules have 'is_reference' set. If provided, the model ID will be used to automatically create the
|
|
31
|
+
new metadata sheet in the Excel file.
|
|
26
32
|
|
|
27
33
|
The following styles are available:
|
|
28
34
|
|
|
@@ -43,12 +49,18 @@ class ExcelExporter(BaseExporter[Workbook]):
|
|
|
43
49
|
}
|
|
44
50
|
style_options = get_args(Style)
|
|
45
51
|
|
|
46
|
-
def __init__(
|
|
52
|
+
def __init__(
|
|
53
|
+
self,
|
|
54
|
+
styling: Style = "default",
|
|
55
|
+
output_role: RoleTypes | None = None,
|
|
56
|
+
new_model_id: tuple[str, str, str] | None = None,
|
|
57
|
+
):
|
|
47
58
|
if styling not in self.style_options:
|
|
48
59
|
raise ValueError(f"Invalid styling: {styling}. Valid options are {self.style_options}")
|
|
49
60
|
self.styling = styling
|
|
50
61
|
self._styling_level = self.style_options.index(styling)
|
|
51
62
|
self.output_role = output_role
|
|
63
|
+
self.new_model_id = new_model_id
|
|
52
64
|
|
|
53
65
|
def export_to_file(self, rules: Rules, filepath: Path) -> None:
|
|
54
66
|
"""Exports transformation rules to excel file."""
|
|
@@ -70,7 +82,7 @@ class ExcelExporter(BaseExporter[Workbook]):
|
|
|
70
82
|
if rules.is_reference:
|
|
71
83
|
# Writes empty reference sheets
|
|
72
84
|
dumped_rules = {
|
|
73
|
-
"Metadata":
|
|
85
|
+
"Metadata": self._create_metadata_sheet_user_rules(rules),
|
|
74
86
|
}
|
|
75
87
|
dumped_rules["Metadata"]["role"] = (
|
|
76
88
|
self.output_role and self.output_role.value
|
|
@@ -155,6 +167,12 @@ class ExcelExporter(BaseExporter[Workbook]):
|
|
|
155
167
|
cell.font = Font(bold=True, size=14)
|
|
156
168
|
|
|
157
169
|
def _write_metadata_sheet(self, workbook: Workbook, metadata: dict[str, Any], is_reference: bool = False) -> None:
|
|
170
|
+
# Excel does not support timezone in datetime strings
|
|
171
|
+
if isinstance(metadata.get("created"), datetime):
|
|
172
|
+
metadata["created"] = metadata["created"].replace(tzinfo=None)
|
|
173
|
+
if isinstance(metadata.get("updated"), datetime):
|
|
174
|
+
metadata["updated"] = metadata["updated"].replace(tzinfo=None)
|
|
175
|
+
|
|
158
176
|
if is_reference:
|
|
159
177
|
metadata_sheet = workbook.create_sheet("RefMetadata")
|
|
160
178
|
else:
|
|
@@ -181,6 +199,7 @@ class ExcelExporter(BaseExporter[Workbook]):
|
|
|
181
199
|
@classmethod
|
|
182
200
|
def _adjust_column_widths(cls, workbook: Workbook) -> None:
|
|
183
201
|
for sheet in workbook:
|
|
202
|
+
sheet = cast(Worksheet, sheet)
|
|
184
203
|
for column_cells in sheet.columns:
|
|
185
204
|
try:
|
|
186
205
|
max_length = max(len(str(cell.value)) for cell in column_cells if cell.value is not None)
|
|
@@ -194,3 +213,74 @@ class ExcelExporter(BaseExporter[Workbook]):
|
|
|
194
213
|
current = sheet.column_dimensions[selected_column.column_letter].width or (max_length + 0.5)
|
|
195
214
|
sheet.column_dimensions[selected_column.column_letter].width = max(current, max_length + 0.5)
|
|
196
215
|
return None
|
|
216
|
+
|
|
217
|
+
def _create_metadata_sheet_user_rules(self, rules: Rules) -> dict[str, Any]:
|
|
218
|
+
metadata: dict[str, Any] = {
|
|
219
|
+
field_alias: None for field_alias in rules.metadata.model_dump(by_alias=True).keys()
|
|
220
|
+
}
|
|
221
|
+
if "creator" in metadata:
|
|
222
|
+
metadata["creator"] = "YOUR NAME"
|
|
223
|
+
|
|
224
|
+
if isinstance(rules, DomainRules):
|
|
225
|
+
return metadata
|
|
226
|
+
elif isinstance(rules, DMSRules):
|
|
227
|
+
existing_model_id = (rules.metadata.space, rules.metadata.external_id, rules.metadata.version)
|
|
228
|
+
elif isinstance(rules, InformationRules):
|
|
229
|
+
existing_model_id = (rules.metadata.prefix, rules.metadata.name, rules.metadata.version)
|
|
230
|
+
else:
|
|
231
|
+
raise ValueError(f"Unsupported rules type: {type(rules)}")
|
|
232
|
+
existing_metadata = rules.metadata.model_dump(by_alias=True)
|
|
233
|
+
if isinstance(existing_metadata["created"], datetime):
|
|
234
|
+
metadata["created"] = existing_metadata["created"].replace(tzinfo=None)
|
|
235
|
+
if isinstance(existing_metadata["updated"], datetime):
|
|
236
|
+
metadata["updated"] = existing_metadata["updated"].replace(tzinfo=None)
|
|
237
|
+
# Excel does not support timezone in datetime strings
|
|
238
|
+
now_iso = datetime.now().replace(tzinfo=None).isoformat()
|
|
239
|
+
is_info = isinstance(rules, InformationRules)
|
|
240
|
+
is_dms = not is_info
|
|
241
|
+
is_extension = self.new_model_id is not None
|
|
242
|
+
is_solution = is_extension and self.new_model_id != existing_model_id
|
|
243
|
+
|
|
244
|
+
if is_solution:
|
|
245
|
+
metadata["prefix" if is_info else "space"] = self.new_model_id[0] # type: ignore[index]
|
|
246
|
+
metadata["title" if is_info else "externalId"] = self.new_model_id[1] # type: ignore[index]
|
|
247
|
+
metadata["version"] = self.new_model_id[2] # type: ignore[index]
|
|
248
|
+
else:
|
|
249
|
+
metadata["prefix" if is_info else "space"] = existing_model_id[0]
|
|
250
|
+
metadata["title" if is_info else "externalId"] = existing_model_id[1]
|
|
251
|
+
metadata["version"] = existing_model_id[2]
|
|
252
|
+
|
|
253
|
+
if is_solution and is_info:
|
|
254
|
+
metadata["namespace"] = f"http://purl.org/{self.new_model_id[0]}/" # type: ignore[index]
|
|
255
|
+
elif is_info:
|
|
256
|
+
metadata["namespace"] = existing_metadata["namespace"]
|
|
257
|
+
|
|
258
|
+
if is_solution and is_dms:
|
|
259
|
+
metadata["name"] = self.new_model_id[1] # type: ignore[index]
|
|
260
|
+
|
|
261
|
+
if is_solution:
|
|
262
|
+
metadata["created"] = now_iso
|
|
263
|
+
else:
|
|
264
|
+
metadata["created"] = existing_metadata["created"]
|
|
265
|
+
|
|
266
|
+
if is_solution or is_extension:
|
|
267
|
+
metadata["updated"] = now_iso
|
|
268
|
+
else:
|
|
269
|
+
metadata["updated"] = existing_metadata["updated"]
|
|
270
|
+
|
|
271
|
+
if is_solution:
|
|
272
|
+
metadata["creator"] = "YOUR NAME"
|
|
273
|
+
else:
|
|
274
|
+
metadata["creator"] = existing_metadata["creator"]
|
|
275
|
+
|
|
276
|
+
if not is_solution:
|
|
277
|
+
metadata["description"] = existing_metadata["description"]
|
|
278
|
+
|
|
279
|
+
if is_extension:
|
|
280
|
+
metadata["schema"] = SchemaCompleteness.extended.value
|
|
281
|
+
else:
|
|
282
|
+
metadata["schema"] = SchemaCompleteness.complete.value
|
|
283
|
+
|
|
284
|
+
metadata["extension"] = "addition"
|
|
285
|
+
|
|
286
|
+
return metadata
|
|
@@ -8,6 +8,7 @@ from pydantic import BaseModel, ConfigDict, ValidationInfo, field_validator
|
|
|
8
8
|
from rdflib import DCTERMS, OWL, RDF, RDFS, XSD, BNode, Graph, Literal, Namespace, URIRef
|
|
9
9
|
from rdflib.collection import Collection as GraphCollection
|
|
10
10
|
|
|
11
|
+
from cognite.neat.constants import DEFAULT_NAMESPACE as NEAT_NAMESPACE
|
|
11
12
|
from cognite.neat.rules import exceptions
|
|
12
13
|
from cognite.neat.rules._analysis._information_rules import InformationArchitectRulesAnalysis
|
|
13
14
|
from cognite.neat.rules.models._rules import DMSRules
|
|
@@ -217,6 +218,7 @@ class OWLMetadata(InformationMetadata):
|
|
|
217
218
|
(URIRef(self.namespace), DCTERMS.hasVersion, Literal(self.version)),
|
|
218
219
|
(URIRef(self.namespace), OWL.versionInfo, Literal(self.version)),
|
|
219
220
|
(URIRef(self.namespace), RDFS.label, Literal(self.name)),
|
|
221
|
+
(URIRef(self.namespace), NEAT_NAMESPACE.prefix, Literal(self.prefix)),
|
|
220
222
|
(URIRef(self.namespace), DCTERMS.title, Literal(self.name)),
|
|
221
223
|
(URIRef(self.namespace), DCTERMS.created, Literal(self.created, datatype=XSD.dateTime)),
|
|
222
224
|
(URIRef(self.namespace), DCTERMS.description, Literal(self.description)),
|
|
@@ -13,7 +13,6 @@ from cognite.client.data_classes.data_modeling import (
|
|
|
13
13
|
SingleHopConnectionDefinition,
|
|
14
14
|
View,
|
|
15
15
|
)
|
|
16
|
-
from cognite.client.data_classes.data_modeling.data_types import ListablePropertyType
|
|
17
16
|
from cognite.client.data_classes.data_modeling.ids import DataModelIdentifier, ViewId
|
|
18
17
|
|
|
19
18
|
from cognite.neat.rules.models.tables import Tables
|
|
@@ -132,9 +131,7 @@ class DMSImporter(BaseImporter):
|
|
|
132
131
|
|
|
133
132
|
max_count: str | float = "1"
|
|
134
133
|
if isinstance(prop, SingleHopConnectionDefinition) or (
|
|
135
|
-
isinstance(prop, MappedProperty)
|
|
136
|
-
and isinstance(prop.type, ListablePropertyType)
|
|
137
|
-
and prop.type.is_list
|
|
134
|
+
isinstance(prop, MappedProperty) and prop.type.is_list
|
|
138
135
|
):
|
|
139
136
|
max_count = float("nan")
|
|
140
137
|
|
|
@@ -5,9 +5,10 @@ from cognite.client import CogniteClient
|
|
|
5
5
|
from cognite.client import data_modeling as dm
|
|
6
6
|
from cognite.client.data_classes.data_modeling import DataModelIdentifier
|
|
7
7
|
from cognite.client.data_classes.data_modeling.containers import BTreeIndex, InvertedIndex
|
|
8
|
-
from cognite.client.
|
|
8
|
+
from cognite.client.utils import ms_to_datetime
|
|
9
9
|
|
|
10
10
|
from cognite.neat.rules import issues
|
|
11
|
+
from cognite.neat.rules.importers._base import BaseImporter, Rules
|
|
11
12
|
from cognite.neat.rules.issues import IssueList
|
|
12
13
|
from cognite.neat.rules.models._rules import DMSRules, DMSSchema, RoleTypes
|
|
13
14
|
from cognite.neat.rules.models._rules._types import (
|
|
@@ -19,6 +20,7 @@ from cognite.neat.rules.models._rules._types import (
|
|
|
19
20
|
ViewEntity,
|
|
20
21
|
ViewPropEntity,
|
|
21
22
|
)
|
|
23
|
+
from cognite.neat.rules.models._rules.base import ExtensionCategory, SchemaCompleteness
|
|
22
24
|
from cognite.neat.rules.models._rules.dms_architect_rules import (
|
|
23
25
|
DMSContainer,
|
|
24
26
|
DMSMetadata,
|
|
@@ -27,16 +29,47 @@ from cognite.neat.rules.models._rules.dms_architect_rules import (
|
|
|
27
29
|
SheetList,
|
|
28
30
|
)
|
|
29
31
|
|
|
30
|
-
from ._base import BaseImporter, Rules
|
|
31
|
-
|
|
32
32
|
|
|
33
33
|
class DMSImporter(BaseImporter):
|
|
34
|
-
def __init__(self, schema: DMSSchema):
|
|
34
|
+
def __init__(self, schema: DMSSchema, metadata: DMSMetadata | None = None):
|
|
35
35
|
self.schema = schema
|
|
36
|
+
self.metadata = metadata
|
|
36
37
|
|
|
37
38
|
@classmethod
|
|
38
39
|
def from_data_model_id(cls, client: CogniteClient, data_model_id: DataModelIdentifier) -> "DMSImporter":
|
|
39
|
-
|
|
40
|
+
"""Create a DMSImporter ready to convert the given data model to rules.
|
|
41
|
+
|
|
42
|
+
Args:
|
|
43
|
+
client: Instantiated CogniteClient to retrieve data model.
|
|
44
|
+
data_model_id: Data Model to retrieve.
|
|
45
|
+
|
|
46
|
+
Returns:
|
|
47
|
+
DMSImporter: DMSImporter instance
|
|
48
|
+
"""
|
|
49
|
+
data_models = client.data_modeling.data_models.retrieve(data_model_id, inline_views=True)
|
|
50
|
+
if len(data_models) == 0:
|
|
51
|
+
raise ValueError(f"Data model {data_model_id} not found")
|
|
52
|
+
data_model = data_models.latest_version()
|
|
53
|
+
schema = DMSSchema.from_data_model(client, data_model)
|
|
54
|
+
description, creator = DMSMetadata._get_description_and_creator(data_model.description)
|
|
55
|
+
|
|
56
|
+
created = ms_to_datetime(data_model.created_time)
|
|
57
|
+
updated = ms_to_datetime(data_model.last_updated_time)
|
|
58
|
+
|
|
59
|
+
metadata = DMSMetadata(
|
|
60
|
+
schema_=SchemaCompleteness.complete,
|
|
61
|
+
extension=ExtensionCategory.addition,
|
|
62
|
+
space=data_model.space,
|
|
63
|
+
external_id=data_model.external_id,
|
|
64
|
+
name=data_model.name or data_model.external_id,
|
|
65
|
+
version=data_model.version or "0.1.0",
|
|
66
|
+
updated=updated,
|
|
67
|
+
created=created,
|
|
68
|
+
creator=creator,
|
|
69
|
+
description=description,
|
|
70
|
+
default_view_version=data_model.version or "0.1.0",
|
|
71
|
+
)
|
|
72
|
+
return cls(schema, metadata)
|
|
40
73
|
|
|
41
74
|
@classmethod
|
|
42
75
|
def from_directory(cls, directory: str | Path) -> "DMSImporter":
|
|
@@ -138,11 +171,7 @@ class DMSImporter(BaseImporter):
|
|
|
138
171
|
description=prop.description,
|
|
139
172
|
value_type=cast(ViewPropEntity | DMSValueType, container_prop.type._type),
|
|
140
173
|
nullable=container_prop.nullable,
|
|
141
|
-
is_list=
|
|
142
|
-
container_prop.type.is_list
|
|
143
|
-
if isinstance(container_prop.type, ListablePropertyType)
|
|
144
|
-
else False
|
|
145
|
-
),
|
|
174
|
+
is_list=container_prop.type.is_list,
|
|
146
175
|
default=container_prop.default_value,
|
|
147
176
|
container=ContainerEntity.from_id(container.as_id()),
|
|
148
177
|
container_property=prop.container_property_identifier,
|
|
@@ -172,7 +201,7 @@ class DMSImporter(BaseImporter):
|
|
|
172
201
|
}
|
|
173
202
|
|
|
174
203
|
dms_rules = DMSRules(
|
|
175
|
-
metadata=DMSMetadata.from_data_model(data_model),
|
|
204
|
+
metadata=self.metadata or DMSMetadata.from_data_model(data_model),
|
|
176
205
|
properties=properties,
|
|
177
206
|
containers=SheetList[DMSContainer](
|
|
178
207
|
data=[DMSContainer.from_container(container) for container in self.schema.containers]
|
|
@@ -3,6 +3,7 @@ import re
|
|
|
3
3
|
|
|
4
4
|
from rdflib import Graph, Namespace
|
|
5
5
|
|
|
6
|
+
from cognite.neat.constants import DEFAULT_NAMESPACE
|
|
6
7
|
from cognite.neat.rules.models._rules.base import RoleTypes, SchemaCompleteness
|
|
7
8
|
from cognite.neat.rules.models.rules import (
|
|
8
9
|
prefix_compliance_regex,
|
|
@@ -29,23 +30,23 @@ def parse_owl_metadata(graph: Graph, make_compliant: bool = False) -> dict:
|
|
|
29
30
|
"""
|
|
30
31
|
# TODO: Move dataframe to dict representation
|
|
31
32
|
|
|
32
|
-
query = """SELECT ?namespace ?prefix ?version ?created ?updated ?title ?description ?creator ?rights ?license
|
|
33
|
-
WHERE {
|
|
33
|
+
query = f"""SELECT ?namespace ?prefix ?version ?created ?updated ?title ?description ?creator ?rights ?license
|
|
34
|
+
WHERE {{
|
|
34
35
|
?namespace a owl:Ontology .
|
|
35
|
-
OPTIONAL {?namespace owl:versionInfo ?version }.
|
|
36
|
-
OPTIONAL {?namespace dcterms:creator ?creator }.
|
|
37
|
-
OPTIONAL {?namespace
|
|
38
|
-
OPTIONAL {?namespace dcterms:
|
|
39
|
-
OPTIONAL {?namespace dcterms:
|
|
40
|
-
OPTIONAL {?namespace dcterms:
|
|
41
|
-
|
|
42
|
-
OPTIONAL {?namespace dcterms:rights|dc:rights ?rights }.
|
|
43
|
-
|
|
44
|
-
OPTIONAL {?namespace dcterms:license|dc:license ?license }.
|
|
36
|
+
OPTIONAL {{?namespace owl:versionInfo ?version }}.
|
|
37
|
+
OPTIONAL {{?namespace dcterms:creator ?creator }}.
|
|
38
|
+
OPTIONAL {{?namespace <{DEFAULT_NAMESPACE.prefix}> ?prefix }}.
|
|
39
|
+
OPTIONAL {{?namespace dcterms:title|rdfs:label|skos:prefLabel ?title }}.
|
|
40
|
+
OPTIONAL {{?namespace dcterms:modified ?updated }}.
|
|
41
|
+
OPTIONAL {{?namespace dcterms:created ?created }}.
|
|
42
|
+
OPTIONAL {{?namespace dcterms:description ?description }}.
|
|
43
|
+
OPTIONAL {{?namespace dcterms:rights|dc:rights ?rights }}.
|
|
44
|
+
|
|
45
|
+
OPTIONAL {{?namespace dcterms:license|dc:license ?license }}.
|
|
45
46
|
FILTER (!isBlank(?namespace))
|
|
46
47
|
FILTER (!bound(?description) || LANG(?description) = "" || LANGMATCHES(LANG(?description), "en"))
|
|
47
48
|
FILTER (!bound(?title) || LANG(?title) = "" || LANGMATCHES(LANG(?title), "en"))
|
|
48
|
-
}
|
|
49
|
+
}}
|
|
49
50
|
"""
|
|
50
51
|
|
|
51
52
|
results = [{item for item in sublist} for sublist in list(zip(*graph.query(query), strict=True))]
|
|
@@ -72,7 +73,6 @@ def parse_owl_metadata(graph: Graph, make_compliant: bool = False) -> dict:
|
|
|
72
73
|
)
|
|
73
74
|
|
|
74
75
|
if make_compliant:
|
|
75
|
-
raw_metadata.pop("created")
|
|
76
76
|
return make_metadata_compliant(raw_metadata)
|
|
77
77
|
|
|
78
78
|
return raw_metadata
|
|
@@ -176,7 +176,7 @@ def fix_date(
|
|
|
176
176
|
if date := metadata.get(date_type, None):
|
|
177
177
|
try:
|
|
178
178
|
if isinstance(date, datetime.datetime):
|
|
179
|
-
|
|
179
|
+
return metadata
|
|
180
180
|
elif isinstance(date, datetime.date):
|
|
181
181
|
metadata[date_type] = datetime.datetime.combine(metadata[date_type], datetime.datetime.min.time())
|
|
182
182
|
elif isinstance(date, str):
|
|
@@ -40,6 +40,8 @@ def parse_owl_properties(graph: Graph, make_compliant: bool = False, language: s
|
|
|
40
40
|
FILTER (!bound(?class) || !isBlank(?class))
|
|
41
41
|
FILTER (!bound(?name) || LANG(?name) = "" || LANGMATCHES(LANG(?name), "en"))
|
|
42
42
|
FILTER (!bound(?description) || LANG(?description) = "" || LANGMATCHES(LANG(?description), "en"))
|
|
43
|
+
BIND(IF(bound(?minCount), ?minCount, 0) AS ?minCount)
|
|
44
|
+
BIND(IF(bound(?maxCount), ?maxCount, 1) AS ?maxCount)
|
|
43
45
|
}
|
|
44
46
|
"""
|
|
45
47
|
|
|
@@ -38,7 +38,7 @@ class OWLImporter(BaseImporter):
|
|
|
38
38
|
|
|
39
39
|
"""
|
|
40
40
|
|
|
41
|
-
def __init__(self, owl_filepath: Path, make_compliant: bool =
|
|
41
|
+
def __init__(self, owl_filepath: Path, make_compliant: bool = False):
|
|
42
42
|
self.owl_filepath = owl_filepath
|
|
43
43
|
self.make_compliant = make_compliant
|
|
44
44
|
|