cognite-neat 0.97.3__py3-none-any.whl → 0.99.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cognite-neat might be problematic. Click here for more details.

Files changed (109) hide show
  1. cognite/neat/_client/__init__.py +4 -0
  2. cognite/neat/_client/_api/data_modeling_loaders.py +512 -0
  3. cognite/neat/_client/_api/schema.py +50 -0
  4. cognite/neat/_client/_api_client.py +17 -0
  5. cognite/neat/_client/data_classes/__init__.py +0 -0
  6. cognite/neat/{_utils/cdf/data_classes.py → _client/data_classes/data_modeling.py} +8 -135
  7. cognite/neat/{_rules/models/dms/_schema.py → _client/data_classes/schema.py} +32 -281
  8. cognite/neat/_graph/_shared.py +14 -15
  9. cognite/neat/_graph/extractors/_classic_cdf/_assets.py +14 -154
  10. cognite/neat/_graph/extractors/_classic_cdf/_base.py +154 -7
  11. cognite/neat/_graph/extractors/_classic_cdf/_classic.py +23 -12
  12. cognite/neat/_graph/extractors/_classic_cdf/_data_sets.py +17 -92
  13. cognite/neat/_graph/extractors/_classic_cdf/_events.py +13 -162
  14. cognite/neat/_graph/extractors/_classic_cdf/_files.py +15 -179
  15. cognite/neat/_graph/extractors/_classic_cdf/_labels.py +32 -100
  16. cognite/neat/_graph/extractors/_classic_cdf/_relationships.py +27 -178
  17. cognite/neat/_graph/extractors/_classic_cdf/_sequences.py +14 -139
  18. cognite/neat/_graph/extractors/_classic_cdf/_timeseries.py +15 -173
  19. cognite/neat/_graph/extractors/_rdf_file.py +6 -7
  20. cognite/neat/_graph/loaders/__init__.py +1 -2
  21. cognite/neat/_graph/queries/_base.py +17 -1
  22. cognite/neat/_graph/transformers/_classic_cdf.py +50 -134
  23. cognite/neat/_graph/transformers/_prune_graph.py +1 -1
  24. cognite/neat/_graph/transformers/_rdfpath.py +1 -1
  25. cognite/neat/_issues/warnings/__init__.py +6 -0
  26. cognite/neat/_issues/warnings/_external.py +8 -0
  27. cognite/neat/_issues/warnings/_models.py +9 -0
  28. cognite/neat/_issues/warnings/_properties.py +16 -0
  29. cognite/neat/_rules/_constants.py +7 -6
  30. cognite/neat/_rules/_shared.py +3 -8
  31. cognite/neat/_rules/analysis/__init__.py +1 -2
  32. cognite/neat/_rules/analysis/_base.py +10 -27
  33. cognite/neat/_rules/analysis/_dms.py +4 -10
  34. cognite/neat/_rules/analysis/_information.py +2 -10
  35. cognite/neat/_rules/catalog/info-rules-imf.xlsx +0 -0
  36. cognite/neat/_rules/exporters/_base.py +3 -4
  37. cognite/neat/_rules/exporters/_rules2dms.py +29 -40
  38. cognite/neat/_rules/exporters/_rules2excel.py +15 -72
  39. cognite/neat/_rules/exporters/_rules2ontology.py +4 -4
  40. cognite/neat/_rules/importers/_base.py +3 -4
  41. cognite/neat/_rules/importers/_dms2rules.py +21 -45
  42. cognite/neat/_rules/importers/_dtdl2rules/dtdl_converter.py +1 -7
  43. cognite/neat/_rules/importers/_dtdl2rules/dtdl_importer.py +7 -10
  44. cognite/neat/_rules/importers/_rdf/_base.py +17 -29
  45. cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2classes.py +2 -2
  46. cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2metadata.py +5 -10
  47. cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2properties.py +1 -2
  48. cognite/neat/_rules/importers/_rdf/_inference2rules.py +55 -51
  49. cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2classes.py +2 -2
  50. cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2metadata.py +5 -8
  51. cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2properties.py +1 -2
  52. cognite/neat/_rules/importers/_rdf/_shared.py +25 -140
  53. cognite/neat/_rules/importers/_spreadsheet2rules.py +10 -41
  54. cognite/neat/_rules/models/__init__.py +3 -17
  55. cognite/neat/_rules/models/_base_rules.py +118 -62
  56. cognite/neat/_rules/models/dms/__init__.py +2 -2
  57. cognite/neat/_rules/models/dms/_exporter.py +20 -178
  58. cognite/neat/_rules/models/dms/_rules.py +65 -128
  59. cognite/neat/_rules/models/dms/_rules_input.py +72 -56
  60. cognite/neat/_rules/models/dms/_validation.py +16 -109
  61. cognite/neat/_rules/models/entities/_single_value.py +32 -4
  62. cognite/neat/_rules/models/information/_rules.py +19 -122
  63. cognite/neat/_rules/models/information/_rules_input.py +32 -41
  64. cognite/neat/_rules/models/information/_validation.py +34 -102
  65. cognite/neat/_rules/models/mapping/__init__.py +2 -3
  66. cognite/neat/_rules/models/mapping/_classic2core.py +36 -146
  67. cognite/neat/_rules/models/mapping/_classic2core.yaml +339 -0
  68. cognite/neat/_rules/transformers/__init__.py +3 -6
  69. cognite/neat/_rules/transformers/_converters.py +128 -206
  70. cognite/neat/_rules/transformers/_mapping.py +105 -34
  71. cognite/neat/_rules/transformers/_verification.py +5 -16
  72. cognite/neat/_session/_base.py +83 -21
  73. cognite/neat/_session/_collector.py +126 -0
  74. cognite/neat/_session/_drop.py +35 -0
  75. cognite/neat/_session/_inspect.py +22 -10
  76. cognite/neat/_session/_mapping.py +39 -0
  77. cognite/neat/_session/_prepare.py +222 -27
  78. cognite/neat/_session/_read.py +109 -19
  79. cognite/neat/_session/_set.py +2 -2
  80. cognite/neat/_session/_show.py +11 -11
  81. cognite/neat/_session/_to.py +27 -14
  82. cognite/neat/_session/exceptions.py +20 -3
  83. cognite/neat/_store/_base.py +27 -24
  84. cognite/neat/_store/_provenance.py +2 -2
  85. cognite/neat/_utils/auxiliary.py +19 -0
  86. cognite/neat/_utils/rdf_.py +28 -1
  87. cognite/neat/_version.py +1 -1
  88. cognite/neat/_workflows/steps/data_contracts.py +2 -10
  89. cognite/neat/_workflows/steps/lib/current/rules_exporter.py +14 -49
  90. cognite/neat/_workflows/steps/lib/current/rules_importer.py +4 -1
  91. cognite/neat/_workflows/steps/lib/current/rules_validator.py +5 -9
  92. {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/METADATA +4 -3
  93. {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/RECORD +97 -100
  94. cognite/neat/_graph/loaders/_rdf2asset.py +0 -416
  95. cognite/neat/_rules/analysis/_asset.py +0 -173
  96. cognite/neat/_rules/models/asset/__init__.py +0 -13
  97. cognite/neat/_rules/models/asset/_rules.py +0 -109
  98. cognite/neat/_rules/models/asset/_rules_input.py +0 -101
  99. cognite/neat/_rules/models/asset/_validation.py +0 -45
  100. cognite/neat/_rules/models/domain.py +0 -136
  101. cognite/neat/_rules/models/mapping/_base.py +0 -131
  102. cognite/neat/_utils/cdf/loaders/__init__.py +0 -25
  103. cognite/neat/_utils/cdf/loaders/_base.py +0 -54
  104. cognite/neat/_utils/cdf/loaders/_data_modeling.py +0 -339
  105. cognite/neat/_utils/cdf/loaders/_ingestion.py +0 -167
  106. /cognite/neat/{_utils/cdf → _client/_api}/__init__.py +0 -0
  107. {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/LICENSE +0 -0
  108. {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/WHEEL +0 -0
  109. {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/entry_points.txt +0 -0
@@ -3,7 +3,6 @@ from collections.abc import Collection, Hashable, Iterable, Sequence
3
3
  from pathlib import Path
4
4
  from typing import Literal, TypeAlias, cast
5
5
 
6
- from cognite.client import CogniteClient
7
6
  from cognite.client.data_classes._base import CogniteResource, CogniteResourceList
8
7
  from cognite.client.data_classes.data_modeling import (
9
8
  ContainerApplyList,
@@ -15,23 +14,14 @@ from cognite.client.data_classes.data_modeling import (
15
14
  )
16
15
  from cognite.client.exceptions import CogniteAPIError
17
16
 
17
+ from cognite.neat._client import DataModelingLoader, NeatClient
18
+ from cognite.neat._client.data_classes.schema import DMSSchema
18
19
  from cognite.neat._issues import IssueList
19
20
  from cognite.neat._issues.warnings import (
20
21
  PrincipleOneModelOneSpaceWarning,
21
22
  ResourceRetrievalWarning,
22
23
  )
23
- from cognite.neat._rules.models.dms import DMSRules, DMSSchema, PipelineSchema
24
- from cognite.neat._utils.cdf.loaders import (
25
- ContainerLoader,
26
- DataModelingLoader,
27
- DataModelLoader,
28
- RawDatabaseLoader,
29
- RawTableLoader,
30
- ResourceLoader,
31
- SpaceLoader,
32
- TransformationLoader,
33
- ViewLoader,
34
- )
24
+ from cognite.neat._rules.models.dms import DMSRules
35
25
  from cognite.neat._utils.upload import UploadResult
36
26
 
37
27
  from ._base import CDFExporter
@@ -119,14 +109,14 @@ class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
119
109
  return rules.as_schema(include_pipeline=self.export_pipeline, instance_space=self.instance_space)
120
110
 
121
111
  def delete_from_cdf(
122
- self, rules: DMSRules, client: CogniteClient, dry_run: bool = False, skip_space: bool = False
112
+ self, rules: DMSRules, client: NeatClient, dry_run: bool = False, skip_space: bool = False
123
113
  ) -> Iterable[UploadResult]:
124
- to_export = self._prepare_exporters(rules, client)
114
+ to_export = self._prepare_exporters(rules)
125
115
 
126
116
  # we need to reverse order in which we are picking up the items to delete
127
117
  # as they are sorted in the order of creation and we need to delete them in reverse order
128
118
  for items, loader in reversed(to_export):
129
- if skip_space and isinstance(loader, SpaceLoader):
119
+ if skip_space and isinstance(items, SpaceApplyList):
130
120
  continue
131
121
  item_ids = loader.get_ids(items)
132
122
  existing_items = loader.retrieve(item_ids)
@@ -166,9 +156,9 @@ class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
166
156
  )
167
157
 
168
158
  def export_to_cdf_iterable(
169
- self, rules: DMSRules, client: CogniteClient, dry_run: bool = False, fallback_one_by_one: bool = False
159
+ self, rules: DMSRules, client: NeatClient, dry_run: bool = False, fallback_one_by_one: bool = False
170
160
  ) -> Iterable[UploadResult]:
171
- to_export = self._prepare_exporters(rules, client)
161
+ to_export = self._prepare_exporters(rules)
172
162
 
173
163
  result_by_name = {}
174
164
  if self.existing_handling == "force":
@@ -176,17 +166,18 @@ class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
176
166
  result_by_name[delete_result.name] = delete_result
177
167
 
178
168
  redeploy_data_model = False
179
- for items, loader in to_export:
169
+ for items in to_export:
180
170
  # The conversion from DMS to GraphQL does not seem to be triggered even if the views
181
171
  # are changed. This is a workaround to force the conversion.
182
- is_redeploying = loader is DataModelingLoader and redeploy_data_model
172
+ is_redeploying = isinstance(items, DataModelApplyList) and redeploy_data_model
173
+ loader = client.loaders.get_loader(items)
183
174
 
184
175
  to_create, to_delete, to_update, unchanged = self._categorize_items_for_upload(
185
176
  loader, items, is_redeploying
186
177
  )
187
178
 
188
179
  issue_list = IssueList()
189
- warning_list = self._validate(loader, items)
180
+ warning_list = self._validate(loader, items, client)
190
181
  issue_list.extend(warning_list)
191
182
 
192
183
  created: set[Hashable] = set()
@@ -206,6 +197,8 @@ class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
206
197
  failed_changed.update(loader.get_id(item) for item in to_update)
207
198
  else:
208
199
  raise ValueError(f"Unsupported existing_handling {self.existing_handling}")
200
+ created.update(loader.get_id(item) for item in to_create)
201
+ deleted.update(loader.get_id(item) for item in to_delete)
209
202
  else:
210
203
  if to_delete:
211
204
  try:
@@ -226,7 +219,7 @@ class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
226
219
  else:
227
220
  deleted.update(loader.get_id(item) for item in to_delete)
228
221
 
229
- if isinstance(loader, DataModelingLoader):
222
+ if isinstance(items, DataModelApplyList):
230
223
  to_create = loader.sort_by_dependencies(to_create)
231
224
 
232
225
  try:
@@ -292,11 +285,11 @@ class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
292
285
  issues=issue_list,
293
286
  )
294
287
 
295
- if loader is ViewLoader and (created or changed):
288
+ if isinstance(items, ViewApplyList) and (created or changed):
296
289
  redeploy_data_model = True
297
290
 
298
291
  def _categorize_items_for_upload(
299
- self, loader: ResourceLoader, items: Sequence[CogniteResource], is_redeploying
292
+ self, loader: DataModelingLoader, items: Sequence[CogniteResource], is_redeploying
300
293
  ) -> tuple[list[CogniteResource], list[CogniteResource], list[CogniteResource], list[CogniteResource]]:
301
294
  item_ids = loader.get_ids(items)
302
295
  cdf_items = loader.retrieve(item_ids)
@@ -304,7 +297,7 @@ class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
304
297
  to_create, to_update, unchanged, to_delete = [], [], [], []
305
298
  for item in items:
306
299
  if (
307
- isinstance(loader, DataModelingLoader)
300
+ isinstance(items, DataModelApplyList)
308
301
  and self.include_space is not None
309
302
  and not loader.in_space(item, self.include_space)
310
303
  ):
@@ -322,28 +315,24 @@ class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
322
315
  to_update.append(item)
323
316
  return to_create, to_delete, to_update, unchanged
324
317
 
325
- def _prepare_exporters(self, rules, client) -> list[tuple[CogniteResourceList, ResourceLoader]]:
318
+ def _prepare_exporters(self, rules: DMSRules) -> list[CogniteResourceList]:
326
319
  schema = self.export(rules)
327
- to_export: list[tuple[CogniteResourceList, ResourceLoader]] = []
320
+ to_export: list[CogniteResourceList] = []
328
321
  if self.export_components.intersection({"all", "spaces"}):
329
- to_export.append((SpaceApplyList(schema.spaces.values()), SpaceLoader(client)))
322
+ to_export.append(SpaceApplyList(schema.spaces.values()))
330
323
  if self.export_components.intersection({"all", "containers"}):
331
- to_export.append((ContainerApplyList(schema.containers.values()), ContainerLoader(client)))
324
+ to_export.append(ContainerApplyList(schema.containers.values()))
332
325
  if self.export_components.intersection({"all", "views"}):
333
- to_export.append((ViewApplyList(schema.views.values()), ViewLoader(client, self.existing_handling)))
326
+ to_export.append(ViewApplyList(schema.views.values()))
334
327
  if self.export_components.intersection({"all", "data_models"}):
335
- to_export.append((DataModelApplyList([schema.data_model]), DataModelLoader(client)))
336
- if isinstance(schema, PipelineSchema):
337
- to_export.append((schema.databases, RawDatabaseLoader(client)))
338
- to_export.append((schema.raw_tables, RawTableLoader(client)))
339
- to_export.append((schema.transformations, TransformationLoader(client)))
328
+ to_export.append(DataModelApplyList([schema.data_model]))
340
329
  return to_export
341
330
 
342
- def _validate(self, loader: ResourceLoader, items: CogniteResourceList) -> IssueList:
331
+ def _validate(self, loader: DataModelingLoader, items: CogniteResourceList, client: NeatClient) -> IssueList:
343
332
  issue_list = IssueList()
344
- if isinstance(loader, DataModelLoader):
333
+ if isinstance(items, DataModelApplyList):
345
334
  models = cast(list[DataModelApply], items)
346
- if other_models := self._exist_other_data_models(loader, models):
335
+ if other_models := self._exist_other_data_models(client, models):
347
336
  warning = PrincipleOneModelOneSpaceWarning(
348
337
  f"There are multiple data models in the same space {models[0].space}. "
349
338
  f"Other data models in the space are {other_models}.",
@@ -355,13 +344,13 @@ class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
355
344
  return issue_list
356
345
 
357
346
  @classmethod
358
- def _exist_other_data_models(cls, loader: DataModelLoader, models: list[DataModelApply]) -> list[DataModelId]:
347
+ def _exist_other_data_models(cls, client: NeatClient, models: list[DataModelApply]) -> list[DataModelId]:
359
348
  if not models:
360
349
  return []
361
350
  space = models[0].space
362
351
  external_id = models[0].external_id
363
352
  try:
364
- data_models = loader.client.data_modeling.data_models.list(space=space, limit=25, all_versions=False)
353
+ data_models = client.data_modeling.data_models.list(space=space, limit=25, all_versions=False)
365
354
  except CogniteAPIError as e:
366
355
  warnings.warn(ResourceRetrievalWarning(frozenset({space}), "space", str(e)), stacklevel=2)
367
356
  return []
@@ -13,17 +13,13 @@ from openpyxl.styles import Alignment, Border, Font, PatternFill, Side
13
13
  from openpyxl.worksheet.worksheet import Worksheet
14
14
  from rdflib import Namespace
15
15
 
16
- from cognite.neat._constants import COGNITE_MODELS
17
16
  from cognite.neat._rules._shared import VerifiedRules
18
17
  from cognite.neat._rules.models import (
19
- DataModelType,
20
18
  ExtensionCategory,
21
19
  SchemaCompleteness,
22
20
  SheetRow,
23
21
  )
24
22
  from cognite.neat._rules.models.dms import DMSMetadata
25
- from cognite.neat._rules.models.dms._rules import DMSRules
26
- from cognite.neat._rules.models.domain import DomainMetadata
27
23
  from cognite.neat._rules.models.information import InformationMetadata
28
24
  from cognite.neat._rules.models.information._rules import InformationRules
29
25
 
@@ -40,23 +36,13 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
40
36
  on the different styles.
41
37
  output_role: The role to use for the exported spreadsheet. If provided, the rules will be converted to
42
38
  this role formate before being written to excel. If not provided, the role from the rules will be used.
43
- dump_as: This determines how the rules are written to the Excel file. An Excel file has up to three sets of
44
- sheets: user, last, and reference. The user sheets are used for inputting rules from a user. The last sheets
45
- are used for the last version of the same model as the user, while the reference sheets are used for
46
- the model the user is building on. The options are:
47
- * "user": The rules are written to the user sheets. This is used when you want to modify the rules
48
- directly and potentially change the model. This is useful when you have imported the data model
49
- from outside CDF and you want to modify it before you write it to CDF.
50
- * "last": The rules are written to the last sheets. This is used when you want to extend the rules,
51
- but have validation that you are not breaking the existing model. This is used when you want to
52
- change a model that has already been published to CDF and that model is in production.
53
- * "reference": The rules are written to the reference sheets. This is typically used when you want to build
54
- a new solution on top of an enterprise model.
55
39
  new_model_id: The new model ID to use for the exported spreadsheet. This is only applicable if the input
56
40
  rules have 'is_reference' set. If provided, the model ID will be used to automatically create the
57
41
  new metadata sheet in the Excel file. The model id is expected to be a tuple of (prefix, title)
58
42
  (space, external_id) for InformationRules and DMSRules respectively.
59
43
 
44
+ sheet_prefix: The prefix to use for the sheet names in the Excel file. Defaults to an empty string.
45
+
60
46
  The following styles are available:
61
47
 
62
48
  - "none": No styling is applied.
@@ -80,16 +66,17 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
80
66
  dump_options = get_args(DumpOptions)
81
67
 
82
68
  def __init__(
83
- self, styling: Style = "default", dump_as: DumpOptions = "user", new_model_id: tuple[str, str] | None = None
69
+ self,
70
+ styling: Style = "default",
71
+ new_model_id: tuple[str, str] | None = None,
72
+ sheet_prefix: str | None = None,
84
73
  ):
74
+ self.sheet_prefix = sheet_prefix or ""
85
75
  if styling not in self.style_options:
86
76
  raise ValueError(f"Invalid styling: {styling}. Valid options are {self.style_options}")
87
- if dump_as not in self.dump_options:
88
- raise ValueError(f"Invalid dump_as: {dump_as}. Valid options are {self.dump_options}")
89
77
  self.styling = styling
90
78
  self._styling_level = self.style_options.index(styling)
91
79
  self.new_model_id = new_model_id
92
- self.dump_as = dump_as
93
80
 
94
81
  def export_to_file(self, rules: VerifiedRules, filepath: Path) -> None:
95
82
  """Exports transformation rules to excel file."""
@@ -105,43 +92,10 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
105
92
  # Remove default sheet named "Sheet"
106
93
  workbook.remove(workbook["Sheet"])
107
94
 
108
- dumped_user_rules: dict[str, Any]
109
- dumped_last_rules: dict[str, Any] | None = None
110
- dumped_reference_rules: dict[str, Any] | None = None
111
- if self.dump_as != "user":
112
- action = {"last": "update", "reference": "create"}[self.dump_as]
113
- metadata_creator = _MetadataCreator(action, self.new_model_id) # type: ignore[arg-type]
114
-
115
- dumped_user_rules = {
116
- "Metadata": metadata_creator.create(rules.metadata),
117
- }
118
-
119
- if self.dump_as == "last":
120
- dumped_last_rules = rules.dump(by_alias=True)
121
- if rules.reference:
122
- dumped_reference_rules = rules.reference.dump(by_alias=True, as_reference=True)
123
- elif self.dump_as == "reference":
124
- dumped_reference_rules = rules.dump(by_alias=True, as_reference=True)
125
- else:
126
- dumped_user_rules = rules.dump(by_alias=True)
127
- if rules.last:
128
- dumped_last_rules = rules.last.dump(by_alias=True)
129
- if rules.reference:
130
- dumped_reference_rules = rules.reference.dump(by_alias=True, as_reference=True)
131
-
132
- self._write_metadata_sheet(workbook, dumped_user_rules["Metadata"])
133
- self._write_sheets(workbook, dumped_user_rules, rules)
134
- if dumped_last_rules:
135
- self._write_sheets(workbook, dumped_last_rules, rules, sheet_prefix="Last")
136
- self._write_metadata_sheet(workbook, dumped_last_rules["Metadata"], sheet_prefix="Last")
137
-
138
- if dumped_reference_rules:
139
- if isinstance(rules.reference, DMSRules):
140
- sheet_prefix = "CDM" if rules.reference.metadata.as_data_model_id() in COGNITE_MODELS else "Ref"
141
- else:
142
- sheet_prefix = "Ref"
143
- self._write_sheets(workbook, dumped_reference_rules, rules, sheet_prefix=sheet_prefix)
144
- self._write_metadata_sheet(workbook, dumped_reference_rules["Metadata"], sheet_prefix=sheet_prefix)
95
+ dumped_user_rules: dict[str, Any] = rules.dump(by_alias=True)
96
+
97
+ self._write_metadata_sheet(workbook, dumped_user_rules["Metadata"], sheet_prefix=self.sheet_prefix)
98
+ self._write_sheets(workbook, dumped_user_rules, rules, sheet_prefix=self.sheet_prefix)
145
99
 
146
100
  if isinstance(rules, InformationRules) and rules.prefixes:
147
101
  self._write_prefixes_sheet(workbook, rules.prefixes)
@@ -274,9 +228,9 @@ class _MetadataCreator:
274
228
  new_model_id: tuple[str, str] | None = None,
275
229
  ):
276
230
  self.action = action
277
- self.new_model_id = new_model_id or ("YOUR_PREFIX", "YOUR_TITLE")
231
+ self.new_model_id = new_model_id or ("YOUR_SPACE", "YOUR_EXTERNAL_ID")
278
232
 
279
- def create(self, metadata: DomainMetadata | InformationMetadata | DMSMetadata) -> dict[str, Any]:
233
+ def create(self, metadata: InformationMetadata | DMSMetadata) -> dict[str, Any]:
280
234
  now = datetime.now(timezone.utc).replace(microsecond=0, tzinfo=None)
281
235
  if self.action == "update":
282
236
  output = json.loads(metadata.model_dump_json(by_alias=True))
@@ -290,13 +244,6 @@ class _MetadataCreator:
290
244
  output["creator"] = self.creator_name
291
245
  return output
292
246
 
293
- # Action "create"
294
- if isinstance(metadata, DomainMetadata):
295
- output = {field_alias: None for field_alias in metadata.model_dump(by_alias=True).keys()}
296
- output["role"] = metadata.role.value
297
- output["creator"] = self.creator_name
298
- return output
299
-
300
247
  new_metadata = self._create_new_info(now)
301
248
  if isinstance(metadata, DMSMetadata):
302
249
  from cognite.neat._rules.transformers._converters import _InformationRulesConverter
@@ -314,13 +261,9 @@ class _MetadataCreator:
314
261
  return created
315
262
 
316
263
  def _create_new_info(self, now: datetime) -> InformationMetadata:
317
- prefix = self.new_model_id[0]
318
264
  return InformationMetadata(
319
- data_model_type=DataModelType.solution,
320
- schema_=SchemaCompleteness.complete,
321
- extension=ExtensionCategory.addition,
322
- prefix=prefix,
323
- namespace=f"http://purl.org/neat/{prefix}/", # type: ignore[arg-type]
265
+ space=self.new_model_id[0],
266
+ external_id=self.new_model_id[1],
324
267
  description=None,
325
268
  version="1",
326
269
  created=now,
@@ -247,9 +247,9 @@ class OWLClass(OntologyModel):
247
247
 
248
248
  @classmethod
249
249
  def from_class(cls, definition: InformationClass, namespace: Namespace, prefixes: dict) -> Self:
250
- if definition.parent and isinstance(definition.parent, list):
250
+ if definition.implements and isinstance(definition.implements, list):
251
251
  sub_class_of = []
252
- for parent_class in definition.parent:
252
+ for parent_class in definition.implements:
253
253
  try:
254
254
  sub_class_of.append(prefixes[str(parent_class.prefix)][str(parent_class.suffix)])
255
255
  except KeyError:
@@ -524,8 +524,8 @@ class SHACLNodeShape(OntologyModel):
524
524
  def from_rules(
525
525
  cls, class_definition: InformationClass, property_definitions: list[InformationProperty], namespace: Namespace
526
526
  ) -> "SHACLNodeShape":
527
- if class_definition.parent:
528
- parent = [namespace[str(parent.suffix) + "Shape"] for parent in class_definition.parent]
527
+ if class_definition.implements:
528
+ parent = [namespace[str(parent.suffix) + "Shape"] for parent in class_definition.implements]
529
529
  else:
530
530
  parent = None
531
531
  return cls(
@@ -6,7 +6,6 @@ from datetime import datetime
6
6
  from typing import Any, Generic, Literal
7
7
 
8
8
  from pydantic import ValidationError
9
- from rdflib import Namespace
10
9
 
11
10
  from cognite.neat._constants import DEFAULT_NAMESPACE
12
11
  from cognite.neat._issues import IssueList, NeatError, NeatWarning
@@ -33,11 +32,11 @@ class BaseImporter(ABC, Generic[T_InputRules]):
33
32
  creator = getpass.getuser()
34
33
 
35
34
  return {
36
- "prefix": "neat",
37
35
  "schema": "partial",
38
- "namespace": Namespace("http://purl.org/cognite/neat/"),
36
+ "space": "neat",
37
+ "external_id": "NeatImportedDataModel",
39
38
  "version": "0.1.0",
40
- "title": "Neat Imported Data Model",
39
+ "name": "Neat Imported Data Model",
41
40
  "created": datetime.now().replace(microsecond=0).isoformat(),
42
41
  "updated": datetime.now().replace(microsecond=0).isoformat(),
43
42
  "creator": creator,
@@ -1,10 +1,9 @@
1
1
  from collections import Counter
2
- from collections.abc import Collection, Sequence
2
+ from collections.abc import Collection, Iterable, Sequence
3
3
  from datetime import datetime, timezone
4
4
  from pathlib import Path
5
5
  from typing import Literal, cast
6
6
 
7
- from cognite.client import CogniteClient
8
7
  from cognite.client import data_modeling as dm
9
8
  from cognite.client.data_classes.data_modeling import DataModelId, DataModelIdentifier
10
9
  from cognite.client.data_classes.data_modeling.containers import BTreeIndex, InvertedIndex
@@ -19,6 +18,7 @@ from cognite.client.data_classes.data_modeling.views import (
19
18
  )
20
19
  from cognite.client.utils import ms_to_datetime
21
20
 
21
+ from cognite.neat._client import NeatClient
22
22
  from cognite.neat._issues import IssueList, NeatIssue
23
23
  from cognite.neat._issues.errors import FileTypeUnexpectedError, ResourceMissingIdentifierError, ResourceRetrievalError
24
24
  from cognite.neat._issues.warnings import (
@@ -30,10 +30,8 @@ from cognite.neat._issues.warnings import (
30
30
  from cognite.neat._rules._shared import ReadRules
31
31
  from cognite.neat._rules.importers._base import BaseImporter, _handle_issues
32
32
  from cognite.neat._rules.models import (
33
- DataModelType,
34
33
  DMSInputRules,
35
34
  DMSSchema,
36
- SchemaCompleteness,
37
35
  )
38
36
  from cognite.neat._rules.models.data_types import DataType, Enum
39
37
  from cognite.neat._rules.models.dms import (
@@ -86,10 +84,18 @@ class DMSImporter(BaseImporter[DMSInputRules]):
86
84
  self._all_containers_by_id.update(schema.reference.containers.items())
87
85
  self._all_views_by_id.update(schema.reference.views.items())
88
86
 
87
+ def update_referenced_containers(self, containers: Iterable[dm.ContainerApply]) -> None:
88
+ """Update the referenced containers. This is useful to add Cognite containers identified after the root schema
89
+ is read"""
90
+ for container in containers:
91
+ if container.as_id() in self._all_containers_by_id:
92
+ continue
93
+ self._all_containers_by_id[container.as_id()] = container
94
+
89
95
  @classmethod
90
96
  def from_data_model_id(
91
97
  cls,
92
- client: CogniteClient,
98
+ client: NeatClient,
93
99
  data_model_id: DataModelIdentifier,
94
100
  reference_model_id: DataModelIdentifier | None = None,
95
101
  ) -> "DMSImporter":
@@ -141,12 +147,12 @@ class DMSImporter(BaseImporter[DMSInputRules]):
141
147
 
142
148
  issue_list = IssueList()
143
149
  with _handle_issues(issue_list) as result:
144
- schema = DMSSchema.from_data_model(client, user_model, ref_model)
150
+ schema = DMSSchema.from_data_model(NeatClient(client), user_model, ref_model)
145
151
 
146
152
  if result.result == "failure" or issue_list.has_errors:
147
153
  return cls(DMSSchema(), issue_list)
148
154
 
149
- metadata = cls._create_metadata_from_model(user_model, has_reference=ref_model is not None)
155
+ metadata = cls._create_metadata_from_model(user_model)
150
156
  ref_metadata = cls._create_metadata_from_model(ref_model) if ref_model else None
151
157
 
152
158
  return cls(schema, issue_list, metadata, ref_metadata)
@@ -168,7 +174,6 @@ class DMSImporter(BaseImporter[DMSInputRules]):
168
174
  def _create_metadata_from_model(
169
175
  cls,
170
176
  model: dm.DataModel[dm.View] | dm.DataModelApply,
171
- has_reference: bool = False,
172
177
  ) -> DMSInputMetadata:
173
178
  description, creator = DMSInputMetadata._get_description_and_creator(model.description)
174
179
 
@@ -180,9 +185,6 @@ class DMSImporter(BaseImporter[DMSInputRules]):
180
185
  created = now
181
186
  updated = now
182
187
  return DMSInputMetadata(
183
- schema_="complete",
184
- data_model_type="solution" if has_reference else "enterprise",
185
- extension="addition",
186
188
  space=model.space,
187
189
  external_id=model.external_id,
188
190
  name=model.name or model.external_id,
@@ -226,28 +228,12 @@ class DMSImporter(BaseImporter[DMSInputRules]):
226
228
 
227
229
  model = self.root_schema.data_model
228
230
 
229
- schema_completeness = SchemaCompleteness.complete
230
- data_model_type = DataModelType.enterprise
231
- reference: DMSInputRules | None = None
232
- if (ref_schema := self.root_schema.reference) and (ref_model := ref_schema.data_model):
233
- # Reference should always be an enterprise model.
234
- reference = self._create_rule_components(
235
- ref_model,
236
- ref_schema,
237
- self.ref_metadata or self._create_default_metadata(list(ref_schema.views.values()), is_ref=True),
238
- DataModelType.enterprise,
239
- )
240
- data_model_type = DataModelType.solution
241
-
242
231
  user_rules = self._create_rule_components(
243
232
  model,
244
233
  self.root_schema,
245
234
  self.metadata,
246
- data_model_type,
247
- schema_completeness,
248
- has_reference=reference is not None,
249
235
  )
250
- user_rules.reference = reference
236
+
251
237
  self._end = datetime.now(timezone.utc)
252
238
 
253
239
  return ReadRules(user_rules, self.issue_list, {})
@@ -257,9 +243,6 @@ class DMSImporter(BaseImporter[DMSInputRules]):
257
243
  data_model: dm.DataModelApply,
258
244
  schema: DMSSchema,
259
245
  metadata: DMSInputMetadata | None = None,
260
- data_model_type: DataModelType | None = None,
261
- schema_completeness: SchemaCompleteness | None = None,
262
- has_reference: bool = False,
263
246
  ) -> DMSInputRules:
264
247
  properties: list[DMSInputProperty] = []
265
248
  for view_id, view in schema.views.items():
@@ -274,11 +257,7 @@ class DMSImporter(BaseImporter[DMSInputRules]):
274
257
  view.as_id() if isinstance(view, dm.View | dm.ViewApply) else view for view in data_model.views or []
275
258
  }
276
259
 
277
- metadata = metadata or DMSInputMetadata.from_data_model(data_model, has_reference)
278
- if data_model_type is not None:
279
- metadata.data_model_type = str(data_model_type) # type: ignore[assignment]
280
- if schema_completeness is not None:
281
- metadata.schema_ = str(schema_completeness) # type: ignore[assignment]
260
+ metadata = metadata or DMSInputMetadata.from_data_model(data_model)
282
261
 
283
262
  enum = self._create_enum_collections(schema.containers.values())
284
263
 
@@ -301,9 +280,6 @@ class DMSImporter(BaseImporter[DMSInputRules]):
301
280
  now = datetime.now().replace(microsecond=0)
302
281
  space = Counter(view.space for view in views).most_common(1)[0][0]
303
282
  return DMSInputMetadata(
304
- schema_="complete",
305
- extension="addition",
306
- data_model_type="enterprise" if is_ref else "solution",
307
283
  space=space,
308
284
  external_id="Unknown",
309
285
  version="0.1.0",
@@ -351,8 +327,6 @@ class DMSImporter(BaseImporter[DMSInputRules]):
351
327
  return None
352
328
 
353
329
  return DMSInputProperty(
354
- class_=str(class_entity),
355
- property_=prop_id,
356
330
  description=prop.description,
357
331
  name=prop.name,
358
332
  connection=self._get_connection_type(prop),
@@ -361,10 +335,12 @@ class DMSImporter(BaseImporter[DMSInputRules]):
361
335
  nullable=self._get_nullable(prop),
362
336
  immutable=self._get_immutable(prop),
363
337
  default=self._get_default(prop),
364
- container=str(ContainerEntity.from_id(prop.container))
365
- if isinstance(prop, dm.MappedPropertyApply)
366
- else None,
367
- container_property=prop.container_property_identifier if isinstance(prop, dm.MappedPropertyApply) else None,
338
+ container=(
339
+ str(ContainerEntity.from_id(prop.container)) if isinstance(prop, dm.MappedPropertyApply) else None
340
+ ),
341
+ container_property=(
342
+ prop.container_property_identifier if isinstance(prop, dm.MappedPropertyApply) else None
343
+ ),
368
344
  view=str(view_entity),
369
345
  view_property=prop_id,
370
346
  index=self._get_index(prop, prop_id),
@@ -99,8 +99,7 @@ class _DTDLConverter:
99
99
  class_=item.id_.as_class_id(),
100
100
  name=item.display_name,
101
101
  description=item.description,
102
- comment=item.comment,
103
- parent=[parent.as_class_id() for parent in item.extends or []] or None,
102
+ implements=[parent.as_class_id() for parent in item.extends or []] or None,
104
103
  )
105
104
  self.classes.append(class_)
106
105
  for sub_item_or_id in item.contents or []:
@@ -135,7 +134,6 @@ class _DTDLConverter:
135
134
  property_=item.name,
136
135
  name=item.display_name,
137
136
  description=item.description,
138
- comment=item.comment,
139
137
  value_type=value_type,
140
138
  min_count=min_count,
141
139
  max_count=max_count,
@@ -181,7 +179,6 @@ class _DTDLConverter:
181
179
  property_=item.name,
182
180
  name=item.display_name,
183
181
  description=item.description,
184
- comment=item.comment,
185
182
  value_type=value_type,
186
183
  min_count=0,
187
184
  max_count=1,
@@ -201,7 +198,6 @@ class _DTDLConverter:
201
198
  property_=item.name,
202
199
  name=item.display_name,
203
200
  description=item.description,
204
- comment=item.comment,
205
201
  value_type=value_type,
206
202
  min_count=0,
207
203
  max_count=1,
@@ -233,7 +229,6 @@ class _DTDLConverter:
233
229
  description=item.description,
234
230
  min_count=item.min_multiplicity or 0,
235
231
  max_count=item.max_multiplicity or 1,
236
- comment=item.comment,
237
232
  value_type=value_type,
238
233
  )
239
234
  self.properties.append(prop)
@@ -255,7 +250,6 @@ class _DTDLConverter:
255
250
  class_=item.id_.as_class_id(),
256
251
  name=item.display_name,
257
252
  description=item.description,
258
- comment=item.comment,
259
253
  )
260
254
  self.classes.append(class_)
261
255
 
@@ -17,7 +17,7 @@ from cognite.neat._rules._shared import ReadRules
17
17
  from cognite.neat._rules.importers._base import BaseImporter
18
18
  from cognite.neat._rules.importers._dtdl2rules.dtdl_converter import _DTDLConverter
19
19
  from cognite.neat._rules.importers._dtdl2rules.spec import DTDL_CLS_BY_TYPE_BY_SPEC, DTDLBase, Interface
20
- from cognite.neat._rules.models import InformationInputRules, SchemaCompleteness
20
+ from cognite.neat._rules.models import InformationInputRules
21
21
  from cognite.neat._rules.models.information import InformationInputMetadata
22
22
  from cognite.neat._utils.text import humanize_collection, to_pascal
23
23
 
@@ -31,7 +31,7 @@ class DTDLImporter(BaseImporter[InformationInputRules]):
31
31
 
32
32
  Args:
33
33
  items (Sequence[DTDLBase]): A sequence of DTDLBase objects.
34
- title (str, optional): Title of the data model. Defaults to None.
34
+ name (str, optional): Name of the data model. Defaults to None.
35
35
  read_issues (list[ValidationIssue], optional): A list of issues that occurred during reading. Defaults to None.
36
36
  schema (SchemaCompleteness, optional): Schema completeness. Defaults to SchemaCompleteness.partial.
37
37
 
@@ -40,14 +40,12 @@ class DTDLImporter(BaseImporter[InformationInputRules]):
40
40
  def __init__(
41
41
  self,
42
42
  items: Sequence[DTDLBase],
43
- title: str | None = None,
43
+ name: str | None = None,
44
44
  read_issues: list[NeatIssue] | None = None,
45
- schema: SchemaCompleteness = SchemaCompleteness.partial,
46
45
  ) -> None:
47
46
  self._items = items
48
- self.title = title
47
+ self.name = name
49
48
  self._read_issues = IssueList(read_issues)
50
- self._schema_completeness = schema
51
49
 
52
50
  @classmethod
53
51
  def _from_file_content(cls, file_content: str, filepath: Path) -> Iterable[DTDLBase | NeatIssue]:
@@ -130,17 +128,16 @@ class DTDLImporter(BaseImporter[InformationInputRules]):
130
128
  converter.convert(self._items)
131
129
 
132
130
  metadata = self._default_metadata()
133
- metadata["schema"] = self._schema_completeness.value
134
131
 
135
- if self.title:
136
- metadata["title"] = to_pascal(self.title)
132
+ if self.name:
133
+ metadata["name"] = to_pascal(self.name)
137
134
  try:
138
135
  most_common_prefix = converter.get_most_common_prefix()
139
136
  except ValueError:
140
137
  # No prefixes are defined so we just use the default prefix...
141
138
  ...
142
139
  else:
143
- metadata["prefix"] = most_common_prefix
140
+ metadata["space"] = most_common_prefix
144
141
 
145
142
  rules = InformationInputRules(
146
143
  metadata=InformationInputMetadata.load(metadata),