cognite-neat 0.97.3__py3-none-any.whl → 0.99.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cognite-neat might be problematic. Click here for more details.

Files changed (109) hide show
  1. cognite/neat/_client/__init__.py +4 -0
  2. cognite/neat/_client/_api/data_modeling_loaders.py +512 -0
  3. cognite/neat/_client/_api/schema.py +50 -0
  4. cognite/neat/_client/_api_client.py +17 -0
  5. cognite/neat/_client/data_classes/__init__.py +0 -0
  6. cognite/neat/{_utils/cdf/data_classes.py → _client/data_classes/data_modeling.py} +8 -135
  7. cognite/neat/{_rules/models/dms/_schema.py → _client/data_classes/schema.py} +32 -281
  8. cognite/neat/_graph/_shared.py +14 -15
  9. cognite/neat/_graph/extractors/_classic_cdf/_assets.py +14 -154
  10. cognite/neat/_graph/extractors/_classic_cdf/_base.py +154 -7
  11. cognite/neat/_graph/extractors/_classic_cdf/_classic.py +23 -12
  12. cognite/neat/_graph/extractors/_classic_cdf/_data_sets.py +17 -92
  13. cognite/neat/_graph/extractors/_classic_cdf/_events.py +13 -162
  14. cognite/neat/_graph/extractors/_classic_cdf/_files.py +15 -179
  15. cognite/neat/_graph/extractors/_classic_cdf/_labels.py +32 -100
  16. cognite/neat/_graph/extractors/_classic_cdf/_relationships.py +27 -178
  17. cognite/neat/_graph/extractors/_classic_cdf/_sequences.py +14 -139
  18. cognite/neat/_graph/extractors/_classic_cdf/_timeseries.py +15 -173
  19. cognite/neat/_graph/extractors/_rdf_file.py +6 -7
  20. cognite/neat/_graph/loaders/__init__.py +1 -2
  21. cognite/neat/_graph/queries/_base.py +17 -1
  22. cognite/neat/_graph/transformers/_classic_cdf.py +50 -134
  23. cognite/neat/_graph/transformers/_prune_graph.py +1 -1
  24. cognite/neat/_graph/transformers/_rdfpath.py +1 -1
  25. cognite/neat/_issues/warnings/__init__.py +6 -0
  26. cognite/neat/_issues/warnings/_external.py +8 -0
  27. cognite/neat/_issues/warnings/_models.py +9 -0
  28. cognite/neat/_issues/warnings/_properties.py +16 -0
  29. cognite/neat/_rules/_constants.py +7 -6
  30. cognite/neat/_rules/_shared.py +3 -8
  31. cognite/neat/_rules/analysis/__init__.py +1 -2
  32. cognite/neat/_rules/analysis/_base.py +10 -27
  33. cognite/neat/_rules/analysis/_dms.py +4 -10
  34. cognite/neat/_rules/analysis/_information.py +2 -10
  35. cognite/neat/_rules/catalog/info-rules-imf.xlsx +0 -0
  36. cognite/neat/_rules/exporters/_base.py +3 -4
  37. cognite/neat/_rules/exporters/_rules2dms.py +29 -40
  38. cognite/neat/_rules/exporters/_rules2excel.py +15 -72
  39. cognite/neat/_rules/exporters/_rules2ontology.py +4 -4
  40. cognite/neat/_rules/importers/_base.py +3 -4
  41. cognite/neat/_rules/importers/_dms2rules.py +21 -45
  42. cognite/neat/_rules/importers/_dtdl2rules/dtdl_converter.py +1 -7
  43. cognite/neat/_rules/importers/_dtdl2rules/dtdl_importer.py +7 -10
  44. cognite/neat/_rules/importers/_rdf/_base.py +17 -29
  45. cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2classes.py +2 -2
  46. cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2metadata.py +5 -10
  47. cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2properties.py +1 -2
  48. cognite/neat/_rules/importers/_rdf/_inference2rules.py +55 -51
  49. cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2classes.py +2 -2
  50. cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2metadata.py +5 -8
  51. cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2properties.py +1 -2
  52. cognite/neat/_rules/importers/_rdf/_shared.py +25 -140
  53. cognite/neat/_rules/importers/_spreadsheet2rules.py +10 -41
  54. cognite/neat/_rules/models/__init__.py +3 -17
  55. cognite/neat/_rules/models/_base_rules.py +118 -62
  56. cognite/neat/_rules/models/dms/__init__.py +2 -2
  57. cognite/neat/_rules/models/dms/_exporter.py +20 -178
  58. cognite/neat/_rules/models/dms/_rules.py +65 -128
  59. cognite/neat/_rules/models/dms/_rules_input.py +72 -56
  60. cognite/neat/_rules/models/dms/_validation.py +16 -109
  61. cognite/neat/_rules/models/entities/_single_value.py +32 -4
  62. cognite/neat/_rules/models/information/_rules.py +19 -122
  63. cognite/neat/_rules/models/information/_rules_input.py +32 -41
  64. cognite/neat/_rules/models/information/_validation.py +34 -102
  65. cognite/neat/_rules/models/mapping/__init__.py +2 -3
  66. cognite/neat/_rules/models/mapping/_classic2core.py +36 -146
  67. cognite/neat/_rules/models/mapping/_classic2core.yaml +339 -0
  68. cognite/neat/_rules/transformers/__init__.py +3 -6
  69. cognite/neat/_rules/transformers/_converters.py +128 -206
  70. cognite/neat/_rules/transformers/_mapping.py +105 -34
  71. cognite/neat/_rules/transformers/_verification.py +5 -16
  72. cognite/neat/_session/_base.py +83 -21
  73. cognite/neat/_session/_collector.py +126 -0
  74. cognite/neat/_session/_drop.py +35 -0
  75. cognite/neat/_session/_inspect.py +22 -10
  76. cognite/neat/_session/_mapping.py +39 -0
  77. cognite/neat/_session/_prepare.py +222 -27
  78. cognite/neat/_session/_read.py +109 -19
  79. cognite/neat/_session/_set.py +2 -2
  80. cognite/neat/_session/_show.py +11 -11
  81. cognite/neat/_session/_to.py +27 -14
  82. cognite/neat/_session/exceptions.py +20 -3
  83. cognite/neat/_store/_base.py +27 -24
  84. cognite/neat/_store/_provenance.py +2 -2
  85. cognite/neat/_utils/auxiliary.py +19 -0
  86. cognite/neat/_utils/rdf_.py +28 -1
  87. cognite/neat/_version.py +1 -1
  88. cognite/neat/_workflows/steps/data_contracts.py +2 -10
  89. cognite/neat/_workflows/steps/lib/current/rules_exporter.py +14 -49
  90. cognite/neat/_workflows/steps/lib/current/rules_importer.py +4 -1
  91. cognite/neat/_workflows/steps/lib/current/rules_validator.py +5 -9
  92. {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/METADATA +4 -3
  93. {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/RECORD +97 -100
  94. cognite/neat/_graph/loaders/_rdf2asset.py +0 -416
  95. cognite/neat/_rules/analysis/_asset.py +0 -173
  96. cognite/neat/_rules/models/asset/__init__.py +0 -13
  97. cognite/neat/_rules/models/asset/_rules.py +0 -109
  98. cognite/neat/_rules/models/asset/_rules_input.py +0 -101
  99. cognite/neat/_rules/models/asset/_validation.py +0 -45
  100. cognite/neat/_rules/models/domain.py +0 -136
  101. cognite/neat/_rules/models/mapping/_base.py +0 -131
  102. cognite/neat/_utils/cdf/loaders/__init__.py +0 -25
  103. cognite/neat/_utils/cdf/loaders/_base.py +0 -54
  104. cognite/neat/_utils/cdf/loaders/_data_modeling.py +0 -339
  105. cognite/neat/_utils/cdf/loaders/_ingestion.py +0 -167
  106. /cognite/neat/{_utils/cdf → _client/_api}/__init__.py +0 -0
  107. {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/LICENSE +0 -0
  108. {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/WHEEL +0 -0
  109. {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/entry_points.txt +0 -0
@@ -1,7 +1,5 @@
1
- import math
2
1
  import warnings
3
2
  from collections.abc import Hashable
4
- from datetime import datetime
5
3
  from typing import Any, ClassVar, Literal
6
4
 
7
5
  import pandas as pd
@@ -10,31 +8,29 @@ from pydantic import Field, field_serializer, field_validator, model_validator
10
8
  from pydantic_core.core_schema import SerializationInfo, ValidationInfo
11
9
  from rdflib import URIRef
12
10
 
13
- from cognite.neat._constants import COGNITE_SPACES, DEFAULT_NAMESPACE
11
+ from cognite.neat._client.data_classes.schema import DMSSchema
12
+ from cognite.neat._constants import COGNITE_SPACES
14
13
  from cognite.neat._issues import MultiValueError
14
+ from cognite.neat._issues.errors import NeatValueError
15
15
  from cognite.neat._issues.warnings import (
16
16
  PrincipleMatchingSpaceAndVersionWarning,
17
- PrincipleSolutionBuildsOnEnterpriseWarning,
18
17
  )
19
18
  from cognite.neat._rules.models._base_rules import (
20
19
  BaseMetadata,
21
20
  BaseRules,
22
- DataModelType,
23
- ExtensionCategory,
21
+ ContainerProperty,
22
+ DataModelAspect,
24
23
  RoleTypes,
25
- SchemaCompleteness,
26
24
  SheetList,
27
25
  SheetRow,
26
+ ViewProperty,
27
+ ViewRef,
28
28
  )
29
29
  from cognite.neat._rules.models._types import (
30
30
  ClassEntityType,
31
31
  ContainerEntityType,
32
- DataModelExternalIdType,
33
32
  DmsPropertyType,
34
- InformationPropertyType,
35
- SpaceType,
36
33
  StrListType,
37
- VersionType,
38
34
  ViewEntityType,
39
35
  )
40
36
  from cognite.neat._rules.models.data_types import DataType
@@ -48,68 +44,18 @@ from cognite.neat._rules.models.entities import (
48
44
  HasDataFilter,
49
45
  NodeTypeFilter,
50
46
  RawFilter,
51
- ReferenceEntity,
52
47
  ReverseConnectionEntity,
53
- URLEntity,
54
48
  ViewEntity,
55
49
  ViewEntityList,
56
50
  )
57
51
 
58
- from ._schema import DMSSchema
59
-
60
52
  _DEFAULT_VERSION = "1"
61
53
 
62
54
 
63
55
  class DMSMetadata(BaseMetadata):
64
56
  role: ClassVar[RoleTypes] = RoleTypes.dms
65
- data_model_type: DataModelType = Field(DataModelType.enterprise, alias="dataModelType")
66
- schema_: SchemaCompleteness = Field(alias="schema")
67
- extension: ExtensionCategory = ExtensionCategory.addition
68
- space: SpaceType
69
- name: str | None = Field(
70
- None,
71
- description="Human readable name of the data model",
72
- min_length=1,
73
- max_length=255,
74
- )
75
- description: str | None = Field(None, min_length=1, max_length=1024)
76
- external_id: DataModelExternalIdType = Field(alias="externalId")
77
- version: VersionType
78
- creator: StrListType
79
- created: datetime = Field(
80
- description=("Date of the data model creation"),
81
- )
82
- updated: datetime = Field(
83
- description=("Date of the data model update"),
84
- )
85
-
86
- @field_validator("*", mode="before")
87
- def strip_string(cls, value: Any) -> Any:
88
- if isinstance(value, str):
89
- return value.strip()
90
- return value
91
-
92
- @field_serializer("schema_", "extension", "data_model_type", when_used="always")
93
- def as_string(self, value: SchemaCompleteness | ExtensionCategory | DataModelType) -> str:
94
- return str(value)
95
-
96
- @field_validator("schema_", mode="plain")
97
- def as_enum_schema(cls, value: str) -> SchemaCompleteness:
98
- return SchemaCompleteness(value.strip())
99
-
100
- @field_validator("extension", mode="plain")
101
- def as_enum_extension(cls, value: str) -> ExtensionCategory:
102
- return ExtensionCategory(value.strip())
103
-
104
- @field_validator("data_model_type", mode="plain")
105
- def as_enum_model_type(cls, value: str) -> DataModelType:
106
- return DataModelType(value.strip())
107
-
108
- @field_validator("description", mode="before")
109
- def nan_as_none(cls, value):
110
- if isinstance(value, float) and math.isnan(value):
111
- return None
112
- return value
57
+ aspect: ClassVar[DataModelAspect] = DataModelAspect.physical
58
+ logical: str | None = None
113
59
 
114
60
  def as_space(self) -> dm.SpaceApply:
115
61
  return dm.SpaceApply(
@@ -159,13 +105,15 @@ class DMSProperty(SheetRow):
159
105
  immutable: bool | None = Field(default=None, alias="Immutable")
160
106
  is_list: bool | None = Field(default=None, alias="Is List")
161
107
  default: str | int | dict | None = Field(None, alias="Default")
162
- reference: URLEntity | ReferenceEntity | None = Field(default=None, alias="Reference", union_mode="left_to_right")
163
108
  container: ContainerEntityType | None = Field(None, alias="Container")
164
109
  container_property: DmsPropertyType | None = Field(None, alias="Container Property")
165
110
  index: StrListType | None = Field(None, alias="Index")
166
111
  constraint: StrListType | None = Field(None, alias="Constraint")
167
- class_: ClassEntityType = Field(alias="Class (linage)")
168
- property_: InformationPropertyType = Field(alias="Property (linage)")
112
+ logical: URIRef | None = Field(
113
+ None,
114
+ alias="Logical",
115
+ description="Used to make connection between physical and logical data model aspect",
116
+ )
169
117
 
170
118
  def _identifier(self) -> tuple[Hashable, ...]:
171
119
  return self.view, self.view_property
@@ -208,19 +156,6 @@ class DMSProperty(SheetRow):
208
156
  )
209
157
  return value
210
158
 
211
- @field_serializer("reference", when_used="always")
212
- def set_reference(self, value: Any, info: SerializationInfo) -> str | None:
213
- if isinstance(info.context, dict) and info.context.get("as_reference") is True:
214
- return str(
215
- ReferenceEntity(
216
- prefix=self.view.prefix,
217
- suffix=self.view.suffix,
218
- version=self.view.version,
219
- property=self.view_property,
220
- )
221
- )
222
- return str(value) if value is not None else None
223
-
224
159
  @field_serializer("value_type", when_used="always")
225
160
  def as_dms_type(self, value_type: DataType | EdgeEntity | ViewEntity, info: SerializationInfo) -> str:
226
161
  if isinstance(value_type, DataType):
@@ -229,7 +164,7 @@ class DMSProperty(SheetRow):
229
164
  return value_type.dump(space=metadata.space, version=metadata.version)
230
165
  return str(value_type)
231
166
 
232
- @field_serializer("view", "container", "class_", when_used="unless-none")
167
+ @field_serializer("view", "container", when_used="unless-none")
233
168
  def remove_default_space(self, value: str, info: SerializationInfo) -> str:
234
169
  if (metadata := _metadata(info.context)) and isinstance(value, Entity):
235
170
  if info.field_name == "container" and info.context.get("as_reference") is True:
@@ -248,15 +183,21 @@ class DMSProperty(SheetRow):
248
183
  return value.dump(space=metadata.space, version=metadata.version, type=default_type)
249
184
  return str(value)
250
185
 
186
+ def as_container_reference(self) -> ContainerProperty:
187
+ if self.container is None or self.container_property is None:
188
+ raise NeatValueError("Accessing container reference without container and container property set")
189
+ return ContainerProperty(container=self.container, property_=self.container_property)
190
+
191
+ def as_view_reference(self) -> ViewProperty:
192
+ return ViewProperty(view=self.view, property_=self.view_property)
193
+
251
194
 
252
195
  class DMSContainer(SheetRow):
253
196
  container: ContainerEntityType = Field(alias="Container")
254
197
  name: str | None = Field(alias="Name", default=None)
255
198
  description: str | None = Field(alias="Description", default=None)
256
- reference: URLEntity | ReferenceEntity | None = Field(alias="Reference", default=None, union_mode="left_to_right")
257
199
  constraint: ContainerEntityList | None = Field(None, alias="Constraint")
258
200
  used_for: Literal["node", "edge", "all"] | None = Field("all", alias="Used For")
259
- class_: ClassEntityType = Field(alias="Class (linage)")
260
201
 
261
202
  def _identifier(self) -> tuple[Hashable, ...]:
262
203
  return (self.container,)
@@ -278,13 +219,7 @@ class DMSContainer(SheetRow):
278
219
  used_for=self.used_for,
279
220
  )
280
221
 
281
- @field_serializer("reference", when_used="always")
282
- def set_reference(self, value: Any, info: SerializationInfo) -> str | None:
283
- if isinstance(info.context, dict) and info.context.get("as_reference") is True:
284
- return self.container.dump()
285
- return str(value) if value is not None else None
286
-
287
- @field_serializer("container", "class_", when_used="unless-none")
222
+ @field_serializer("container", when_used="unless-none")
288
223
  def remove_default_space(self, value: Any, info: SerializationInfo) -> str:
289
224
  if metadata := _metadata(info.context):
290
225
  if isinstance(value, DMSEntity):
@@ -310,21 +245,18 @@ class DMSView(SheetRow):
310
245
  name: str | None = Field(alias="Name", default=None)
311
246
  description: str | None = Field(alias="Description", default=None)
312
247
  implements: ViewEntityList | None = Field(None, alias="Implements")
313
- reference: URLEntity | ReferenceEntity | None = Field(alias="Reference", default=None, union_mode="left_to_right")
314
248
  filter_: HasDataFilter | NodeTypeFilter | RawFilter | None = Field(None, alias="Filter")
315
249
  in_model: bool = Field(True, alias="In Model")
316
- class_: ClassEntityType = Field(alias="Class (linage)")
250
+ logical: URIRef | None = Field(
251
+ None,
252
+ alias="Logical",
253
+ description="Used to make connection between physical and logical data model aspect",
254
+ )
317
255
 
318
256
  def _identifier(self) -> tuple[Hashable, ...]:
319
257
  return (self.view,)
320
258
 
321
- @field_serializer("reference", when_used="always")
322
- def set_reference(self, value: Any, info: SerializationInfo) -> str | None:
323
- if isinstance(info.context, dict) and info.context.get("as_reference") is True:
324
- return self.view.dump()
325
- return str(value) if value is not None else None
326
-
327
- @field_serializer("view", "class_", when_used="unless-none")
259
+ @field_serializer("view", when_used="unless-none")
328
260
  def remove_default_space(self, value: Any, info: SerializationInfo) -> str:
329
261
  if (metadata := _metadata(info.context)) and isinstance(value, Entity):
330
262
  return value.dump(prefix=metadata.space, version=metadata.version)
@@ -344,11 +276,6 @@ class DMSView(SheetRow):
344
276
  def as_view(self) -> dm.ViewApply:
345
277
  view_id = self.view.as_id()
346
278
  implements = [parent.as_id() for parent in self.implements or []] or None
347
- if implements is None and isinstance(self.reference, ReferenceEntity):
348
- # Fallback to the reference if no implements are provided
349
- parent = self.reference.as_view_id()
350
- if (parent.space, parent.external_id) != (view_id.space, view_id.external_id):
351
- implements = [parent]
352
279
 
353
280
  return dm.ViewApply(
354
281
  space=view_id.space,
@@ -356,10 +283,14 @@ class DMSView(SheetRow):
356
283
  version=view_id.version or _DEFAULT_VERSION,
357
284
  name=self.name or None,
358
285
  description=self.description,
286
+ filter=None if self.filter_ is None else self.filter_.as_dms_filter(),
359
287
  implements=implements,
360
288
  properties={},
361
289
  )
362
290
 
291
+ def as_view_reference(self) -> ViewRef:
292
+ return ViewRef(view=self.view)
293
+
363
294
 
364
295
  class DMSNode(SheetRow):
365
296
  node: DMSNodeEntity = Field(alias="Node")
@@ -408,24 +339,9 @@ class DMSRules(BaseRules):
408
339
  containers: SheetList[DMSContainer] | None = Field(None, alias="Containers")
409
340
  enum: SheetList[DMSEnum] | None = Field(None, alias="Enum")
410
341
  nodes: SheetList[DMSNode] | None = Field(None, alias="Nodes")
411
- last: "DMSRules | None" = Field(None, alias="Last", description="The previous version of the data model")
412
- reference: "DMSRules | None" = Field(None, alias="Reference")
413
-
414
- @field_validator("reference")
415
- def check_reference_of_reference(cls, value: "DMSRules | None", info: ValidationInfo) -> "DMSRules | None":
416
- if value is None:
417
- return None
418
- if value.reference is not None:
419
- raise ValueError("Reference rules cannot have a reference")
420
- if value.metadata.data_model_type == DataModelType.solution and (metadata := info.data.get("metadata")):
421
- warnings.warn(
422
- PrincipleSolutionBuildsOnEnterpriseWarning(
423
- f"The solution model {metadata.as_data_model_id()} is referencing another "
424
- f"solution model {value.metadata.as_data_model_id()}",
425
- ),
426
- stacklevel=2,
427
- )
428
- return value
342
+ # This is a hack to allow the post_validation to be turned off when needed
343
+ # Will likely be moved completely out of the rules in the future
344
+ post_validate: bool = Field(default=True, exclude=True, repr=False)
429
345
 
430
346
  @field_validator("views")
431
347
  def matching_version_and_space(cls, value: SheetList[DMSView], info: ValidationInfo) -> SheetList[DMSView]:
@@ -462,6 +378,8 @@ class DMSRules(BaseRules):
462
378
  def post_validation(self) -> "DMSRules":
463
379
  from ._validation import DMSPostValidation
464
380
 
381
+ if not self.post_validate:
382
+ return self
465
383
  issue_list = DMSPostValidation(self).validate()
466
384
  if issue_list.warnings:
467
385
  issue_list.trigger_warnings()
@@ -472,11 +390,11 @@ class DMSRules(BaseRules):
472
390
  def as_schema(self, include_pipeline: bool = False, instance_space: str | None = None) -> DMSSchema:
473
391
  from ._exporter import _DMSExporter
474
392
 
475
- return _DMSExporter(self, include_pipeline, instance_space).to_schema()
393
+ return _DMSExporter(self, instance_space).to_schema()
476
394
 
477
395
  def _repr_html_(self) -> str:
478
396
  summary = {
479
- "type": "Physical Data Model",
397
+ "aspect": self.metadata.aspect,
480
398
  "intended for": "DMS Architect",
481
399
  "name": self.metadata.name,
482
400
  "space": self.metadata.space,
@@ -489,8 +407,27 @@ class DMSRules(BaseRules):
489
407
 
490
408
  return pd.DataFrame([summary]).T.rename(columns={0: ""})._repr_html_() # type: ignore
491
409
 
492
- @property
493
- def id_(self) -> URIRef:
494
- return DEFAULT_NAMESPACE[
495
- f"data-model/verified/dms/{self.metadata.space}/{self.metadata.external_id}/{self.metadata.version}"
496
- ]
410
+ def imported_views_and_containers_ids(
411
+ self, include_model_views_with_no_properties: bool = True
412
+ ) -> tuple[set[dm.ViewId], set[dm.ContainerId]]:
413
+ existing_views = {view.view for view in self.views}
414
+ imported_views: set[dm.ViewId] = set()
415
+ for view in self.views:
416
+ for parent in view.implements or []:
417
+ if parent not in existing_views:
418
+ imported_views.add(parent.as_id())
419
+ existing_containers = {container.container for container in self.containers or []}
420
+ imported_containers: set[dm.ContainerId] = set()
421
+ view_with_properties: set[ViewEntity] = set()
422
+ for prop in self.properties:
423
+ if prop.container and prop.container not in existing_containers:
424
+ imported_containers.add(prop.container.as_id())
425
+ if prop.view not in existing_views:
426
+ imported_views.add(prop.view.as_id())
427
+ view_with_properties.add(prop.view)
428
+
429
+ if include_model_views_with_no_properties:
430
+ extra_views = existing_views - view_with_properties
431
+ imported_views.update({view.as_id() for view in extra_views})
432
+
433
+ return imported_views, imported_containers
@@ -5,13 +5,13 @@ from typing import Any, Literal
5
5
 
6
6
  import pandas as pd
7
7
  from cognite.client import data_modeling as dm
8
- from rdflib import URIRef
8
+ from cognite.client.data_classes.data_modeling import ContainerId, ViewId
9
+ from rdflib import Namespace, URIRef
9
10
 
10
11
  from cognite.neat._constants import DEFAULT_NAMESPACE
11
12
  from cognite.neat._rules.models._base_input import InputComponent, InputRules
12
13
  from cognite.neat._rules.models.data_types import DataType
13
14
  from cognite.neat._rules.models.entities import (
14
- ClassEntity,
15
15
  ContainerEntity,
16
16
  DMSNodeEntity,
17
17
  DMSUnknownEntity,
@@ -27,17 +27,15 @@ from ._rules import _DEFAULT_VERSION, DMSContainer, DMSEnum, DMSMetadata, DMSNod
27
27
 
28
28
  @dataclass
29
29
  class DMSInputMetadata(InputComponent[DMSMetadata]):
30
- schema_: Literal["complete", "partial", "extended"]
31
30
  space: str
32
31
  external_id: str
33
32
  creator: str
34
33
  version: str
35
- extension: Literal["addition", "reshape", "rebuild"] = "addition"
36
- data_model_type: Literal["solution", "enterprise"] = "solution"
37
34
  name: str | None = None
38
35
  description: str | None = None
39
36
  created: datetime | str | None = None
40
37
  updated: datetime | str | None = None
38
+ logical: str | None = None
41
39
 
42
40
  @classmethod
43
41
  def _get_verified_cls(cls) -> type[DMSMetadata]:
@@ -52,11 +50,9 @@ class DMSInputMetadata(InputComponent[DMSMetadata]):
52
50
  return output
53
51
 
54
52
  @classmethod
55
- def from_data_model(cls, data_model: dm.DataModelApply, has_reference: bool) -> "DMSInputMetadata":
53
+ def from_data_model(cls, data_model: dm.DataModelApply) -> "DMSInputMetadata":
56
54
  description, creator = cls._get_description_and_creator(data_model.description)
57
55
  return cls(
58
- schema_="complete",
59
- data_model_type="solution" if has_reference else "enterprise",
60
56
  space=data_model.space,
61
57
  name=data_model.name or None,
62
58
  description=description,
@@ -80,14 +76,27 @@ class DMSInputMetadata(InputComponent[DMSMetadata]):
80
76
  description = None
81
77
  return description, creator
82
78
 
79
+ @property
80
+ def identifier(self) -> URIRef:
81
+ """Globally unique identifier for the data model.
82
+
83
+ !!! note
84
+ Unlike namespace, the identifier does not end with "/" or "#".
85
+
86
+ """
87
+ return DEFAULT_NAMESPACE[f"data-model/unverified/physical/{self.space}/{self.external_id}/{self.version}"]
88
+
89
+ @property
90
+ def namespace(self) -> Namespace:
91
+ """Namespace for the data model used for the entities in the data model."""
92
+ return Namespace(f"{self.identifier}/")
93
+
83
94
 
84
95
  @dataclass
85
96
  class DMSInputProperty(InputComponent[DMSProperty]):
86
97
  view: str
87
98
  view_property: str | None
88
99
  value_type: str | DataType | ViewEntity | DMSUnknownEntity
89
- property_: str | None = None
90
- class_: str | None = None
91
100
  name: str | None = None
92
101
  description: str | None = None
93
102
  connection: Literal["direct"] | ReverseConnectionEntity | EdgeEntity | str | None = None
@@ -95,11 +104,11 @@ class DMSInputProperty(InputComponent[DMSProperty]):
95
104
  immutable: bool | None = None
96
105
  is_list: bool | None = None
97
106
  default: str | int | dict | None = None
98
- reference: str | None = None
99
107
  container: str | None = None
100
108
  container_property: str | None = None
101
109
  index: str | list[str] | None = None
102
110
  constraint: str | list[str] | None = None
111
+ logical: str | None = None
103
112
 
104
113
  @classmethod
105
114
  def _get_verified_cls(cls) -> type[DMSProperty]:
@@ -110,12 +119,6 @@ class DMSInputProperty(InputComponent[DMSProperty]):
110
119
  output["View"] = ViewEntity.load(self.view, space=default_space, version=default_version)
111
120
  output["Value Type"] = load_dms_value_type(self.value_type, default_space, default_version)
112
121
  output["Connection"] = load_connection(self.connection, default_space, default_version)
113
- output["Property (linage)"] = self.property_ or self.view_property
114
- output["Class (linage)"] = (
115
- ClassEntity.load(self.class_ or self.view, prefix=default_space, version=default_version)
116
- if self.class_ or self.view
117
- else None
118
- )
119
122
  output["Container"] = (
120
123
  ContainerEntity.load(self.container, space=default_space, version=default_version)
121
124
  if self.container
@@ -123,14 +126,18 @@ class DMSInputProperty(InputComponent[DMSProperty]):
123
126
  )
124
127
  return output
125
128
 
129
+ def referenced_view(self, default_space: str, default_version: str) -> ViewEntity:
130
+ return ViewEntity.load(self.view, strict=True, space=default_space, version=default_version)
131
+
132
+ def referenced_container(self, default_space: str) -> ContainerEntity | None:
133
+ return ContainerEntity.load(self.container, strict=True, space=default_space) if self.container else None
134
+
126
135
 
127
136
  @dataclass
128
137
  class DMSInputContainer(InputComponent[DMSContainer]):
129
138
  container: str
130
- class_: str | None = None
131
139
  name: str | None = None
132
140
  description: str | None = None
133
- reference: str | None = None
134
141
  constraint: str | None = None
135
142
  used_for: Literal["node", "edge", "all"] | None = None
136
143
 
@@ -140,11 +147,7 @@ class DMSInputContainer(InputComponent[DMSContainer]):
140
147
 
141
148
  def dump(self, default_space: str) -> dict[str, Any]: # type: ignore[override]
142
149
  output = super().dump()
143
- container = ContainerEntity.load(self.container, space=default_space)
144
- output["Container"] = container
145
- output["Class (linage)"] = (
146
- ClassEntity.load(self.class_, prefix=default_space) if self.class_ else container.as_class()
147
- )
150
+ output["Container"] = self.as_entity_id(default_space)
148
151
  output["Constraint"] = (
149
152
  [ContainerEntity.load(constraint.strip(), space=default_space) for constraint in self.constraint.split(",")]
150
153
  if self.constraint
@@ -152,6 +155,9 @@ class DMSInputContainer(InputComponent[DMSContainer]):
152
155
  )
153
156
  return output
154
157
 
158
+ def as_entity_id(self, default_space: str) -> ContainerEntity:
159
+ return ContainerEntity.load(self.container, strict=True, space=default_space)
160
+
155
161
  @classmethod
156
162
  def from_container(cls, container: dm.ContainerApply) -> "DMSInputContainer":
157
163
  constraints: list[str] = []
@@ -161,7 +167,6 @@ class DMSInputContainer(InputComponent[DMSContainer]):
161
167
  # UniquenessConstraint it handled in the properties
162
168
  container_entity = ContainerEntity.from_id(container.as_id())
163
169
  return cls(
164
- class_=str(container_entity.as_class()),
165
170
  container=str(container_entity),
166
171
  name=container.name or None,
167
172
  description=container.description,
@@ -173,13 +178,12 @@ class DMSInputContainer(InputComponent[DMSContainer]):
173
178
  @dataclass
174
179
  class DMSInputView(InputComponent[DMSView]):
175
180
  view: str
176
- class_: str | None = None
177
181
  name: str | None = None
178
182
  description: str | None = None
179
183
  implements: str | None = None
180
- reference: str | None = None
181
- filter_: Literal["hasData", "nodeType", "rawFilter"] | None = None
184
+ filter_: Literal["hasData", "nodeType", "rawFilter"] | str | None = None
182
185
  in_model: bool = True
186
+ logical: str | None = None
183
187
 
184
188
  @classmethod
185
189
  def _get_verified_cls(cls) -> type[DMSView]:
@@ -187,30 +191,31 @@ class DMSInputView(InputComponent[DMSView]):
187
191
 
188
192
  def dump(self, default_space: str, default_version: str) -> dict[str, Any]: # type: ignore[override]
189
193
  output = super().dump()
190
- view = ViewEntity.load(self.view, space=default_space, version=default_version)
191
- output["View"] = view
192
- output["Class (linage)"] = (
193
- ClassEntity.load(self.class_, prefix=default_space, version=default_version)
194
- if self.class_
195
- else view.as_class()
196
- )
197
- output["Implements"] = (
194
+ output["View"] = self.as_entity_id(default_space, default_version)
195
+ output["Implements"] = self._load_implements(default_space, default_version)
196
+ return output
197
+
198
+ def as_entity_id(self, default_space: str, default_version: str) -> ViewEntity:
199
+ return ViewEntity.load(self.view, strict=True, space=default_space, version=default_version)
200
+
201
+ def _load_implements(self, default_space: str, default_version: str) -> list[ViewEntity] | None:
202
+ return (
198
203
  [
199
- ViewEntity.load(implement, space=default_space, version=default_version)
204
+ ViewEntity.load(implement, strict=True, space=default_space, version=default_version)
200
205
  for implement in self.implements.split(",")
201
206
  ]
202
207
  if self.implements
203
208
  else None
204
209
  )
205
- return output
210
+
211
+ def referenced_views(self, default_space: str, default_version: str) -> list[ViewEntity]:
212
+ return self._load_implements(default_space, default_version) or []
206
213
 
207
214
  @classmethod
208
215
  def from_view(cls, view: dm.ViewApply, in_model: bool) -> "DMSInputView":
209
216
  view_entity = ViewEntity.from_id(view.as_id())
210
- class_entity = view_entity.as_class(skip_version=True)
211
217
 
212
218
  return cls(
213
- class_=str(class_entity),
214
219
  view=str(view_entity),
215
220
  description=view.description,
216
221
  name=view.name,
@@ -261,8 +266,6 @@ class DMSInputRules(InputRules[DMSRules]):
261
266
  containers: list[DMSInputContainer] | None = None
262
267
  enum: list[DMSInputEnum] | None = None
263
268
  nodes: list[DMSInputNode] | None = None
264
- last: "DMSInputRules | None" = None
265
- reference: "DMSInputRules | None" = None
266
269
 
267
270
  @classmethod
268
271
  def _get_verified_cls(cls) -> type[DMSRules]:
@@ -271,18 +274,6 @@ class DMSInputRules(InputRules[DMSRules]):
271
274
  def dump(self) -> dict[str, Any]:
272
275
  default_space = self.metadata.space
273
276
  default_version = str(self.metadata.version)
274
- reference: dict[str, Any] | None = None
275
- if isinstance(self.reference, DMSInputRules):
276
- reference = self.reference.dump()
277
- elif isinstance(self.reference, DMSRules):
278
- # We need to load through the DMSRulesInput to set the correct default space and version
279
- reference = DMSInputRules.load(self.reference.model_dump()).dump()
280
- last: dict[str, Any] | None = None
281
- if isinstance(self.last, DMSInputRules):
282
- last = self.last.dump()
283
- elif isinstance(self.last, DMSRules):
284
- # We need to load through the DMSRulesInput to set the correct default space and version
285
- last = DMSInputRules.load(self.last.model_dump()).dump()
286
277
 
287
278
  return {
288
279
  "Metadata": self.metadata.dump(),
@@ -291,8 +282,6 @@ class DMSInputRules(InputRules[DMSRules]):
291
282
  "Containers": [container.dump(default_space) for container in self.containers or []] or None,
292
283
  "Enum": [enum.dump() for enum in self.enum or []] or None,
293
284
  "Nodes": [node_type.dump(default_space) for node_type in self.nodes or []] or None,
294
- "Last": last,
295
- "Reference": reference,
296
285
  }
297
286
 
298
287
  def _repr_html_(self) -> str:
@@ -315,3 +304,30 @@ class DMSInputRules(InputRules[DMSRules]):
315
304
  return DEFAULT_NAMESPACE[
316
305
  f"data-model/unverified/dms/{self.metadata.space}/{self.metadata.external_id}/{self.metadata.version}"
317
306
  ]
307
+
308
+ def referenced_views_and_containers(self) -> tuple[set[ViewEntity], set[ContainerEntity]]:
309
+ default_space = self.metadata.space
310
+ default_version = self.metadata.version
311
+
312
+ containers: set[ContainerEntity] = set()
313
+ views = {parent for view in self.views for parent in view.referenced_views(default_space, default_version)}
314
+ for prop in self.properties:
315
+ views.add(prop.referenced_view(default_space, default_version))
316
+ if ref_container := prop.referenced_container(default_space):
317
+ containers.add(ref_container)
318
+
319
+ return views, containers
320
+
321
+ def as_view_entities(self) -> list[ViewEntity]:
322
+ return [view.as_entity_id(self.metadata.space, self.metadata.version) for view in self.views]
323
+
324
+ def as_container_entities(self) -> list[ContainerEntity]:
325
+ return [container.as_entity_id(self.metadata.space) for container in self.containers or []]
326
+
327
+ def imported_views_and_containers(self) -> tuple[set[ViewEntity], set[ContainerEntity]]:
328
+ views, containers = self.referenced_views_and_containers()
329
+ return views - set(self.as_view_entities()), containers - set(self.as_container_entities())
330
+
331
+ def imported_views_and_containers_ids(self) -> tuple[set[ViewId], set[ContainerId]]:
332
+ views, containers = self.imported_views_and_containers()
333
+ return {view.as_id() for view in views}, {container.as_id() for container in containers}