cognite-neat 0.97.3__py3-none-any.whl → 0.99.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cognite-neat might be problematic. Click here for more details.

Files changed (109) hide show
  1. cognite/neat/_client/__init__.py +4 -0
  2. cognite/neat/_client/_api/data_modeling_loaders.py +512 -0
  3. cognite/neat/_client/_api/schema.py +50 -0
  4. cognite/neat/_client/_api_client.py +17 -0
  5. cognite/neat/_client/data_classes/__init__.py +0 -0
  6. cognite/neat/{_utils/cdf/data_classes.py → _client/data_classes/data_modeling.py} +8 -135
  7. cognite/neat/{_rules/models/dms/_schema.py → _client/data_classes/schema.py} +32 -281
  8. cognite/neat/_graph/_shared.py +14 -15
  9. cognite/neat/_graph/extractors/_classic_cdf/_assets.py +14 -154
  10. cognite/neat/_graph/extractors/_classic_cdf/_base.py +154 -7
  11. cognite/neat/_graph/extractors/_classic_cdf/_classic.py +23 -12
  12. cognite/neat/_graph/extractors/_classic_cdf/_data_sets.py +17 -92
  13. cognite/neat/_graph/extractors/_classic_cdf/_events.py +13 -162
  14. cognite/neat/_graph/extractors/_classic_cdf/_files.py +15 -179
  15. cognite/neat/_graph/extractors/_classic_cdf/_labels.py +32 -100
  16. cognite/neat/_graph/extractors/_classic_cdf/_relationships.py +27 -178
  17. cognite/neat/_graph/extractors/_classic_cdf/_sequences.py +14 -139
  18. cognite/neat/_graph/extractors/_classic_cdf/_timeseries.py +15 -173
  19. cognite/neat/_graph/extractors/_rdf_file.py +6 -7
  20. cognite/neat/_graph/loaders/__init__.py +1 -2
  21. cognite/neat/_graph/queries/_base.py +17 -1
  22. cognite/neat/_graph/transformers/_classic_cdf.py +50 -134
  23. cognite/neat/_graph/transformers/_prune_graph.py +1 -1
  24. cognite/neat/_graph/transformers/_rdfpath.py +1 -1
  25. cognite/neat/_issues/warnings/__init__.py +6 -0
  26. cognite/neat/_issues/warnings/_external.py +8 -0
  27. cognite/neat/_issues/warnings/_models.py +9 -0
  28. cognite/neat/_issues/warnings/_properties.py +16 -0
  29. cognite/neat/_rules/_constants.py +7 -6
  30. cognite/neat/_rules/_shared.py +3 -8
  31. cognite/neat/_rules/analysis/__init__.py +1 -2
  32. cognite/neat/_rules/analysis/_base.py +10 -27
  33. cognite/neat/_rules/analysis/_dms.py +4 -10
  34. cognite/neat/_rules/analysis/_information.py +2 -10
  35. cognite/neat/_rules/catalog/info-rules-imf.xlsx +0 -0
  36. cognite/neat/_rules/exporters/_base.py +3 -4
  37. cognite/neat/_rules/exporters/_rules2dms.py +29 -40
  38. cognite/neat/_rules/exporters/_rules2excel.py +15 -72
  39. cognite/neat/_rules/exporters/_rules2ontology.py +4 -4
  40. cognite/neat/_rules/importers/_base.py +3 -4
  41. cognite/neat/_rules/importers/_dms2rules.py +21 -45
  42. cognite/neat/_rules/importers/_dtdl2rules/dtdl_converter.py +1 -7
  43. cognite/neat/_rules/importers/_dtdl2rules/dtdl_importer.py +7 -10
  44. cognite/neat/_rules/importers/_rdf/_base.py +17 -29
  45. cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2classes.py +2 -2
  46. cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2metadata.py +5 -10
  47. cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2properties.py +1 -2
  48. cognite/neat/_rules/importers/_rdf/_inference2rules.py +55 -51
  49. cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2classes.py +2 -2
  50. cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2metadata.py +5 -8
  51. cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2properties.py +1 -2
  52. cognite/neat/_rules/importers/_rdf/_shared.py +25 -140
  53. cognite/neat/_rules/importers/_spreadsheet2rules.py +10 -41
  54. cognite/neat/_rules/models/__init__.py +3 -17
  55. cognite/neat/_rules/models/_base_rules.py +118 -62
  56. cognite/neat/_rules/models/dms/__init__.py +2 -2
  57. cognite/neat/_rules/models/dms/_exporter.py +20 -178
  58. cognite/neat/_rules/models/dms/_rules.py +65 -128
  59. cognite/neat/_rules/models/dms/_rules_input.py +72 -56
  60. cognite/neat/_rules/models/dms/_validation.py +16 -109
  61. cognite/neat/_rules/models/entities/_single_value.py +32 -4
  62. cognite/neat/_rules/models/information/_rules.py +19 -122
  63. cognite/neat/_rules/models/information/_rules_input.py +32 -41
  64. cognite/neat/_rules/models/information/_validation.py +34 -102
  65. cognite/neat/_rules/models/mapping/__init__.py +2 -3
  66. cognite/neat/_rules/models/mapping/_classic2core.py +36 -146
  67. cognite/neat/_rules/models/mapping/_classic2core.yaml +339 -0
  68. cognite/neat/_rules/transformers/__init__.py +3 -6
  69. cognite/neat/_rules/transformers/_converters.py +128 -206
  70. cognite/neat/_rules/transformers/_mapping.py +105 -34
  71. cognite/neat/_rules/transformers/_verification.py +5 -16
  72. cognite/neat/_session/_base.py +83 -21
  73. cognite/neat/_session/_collector.py +126 -0
  74. cognite/neat/_session/_drop.py +35 -0
  75. cognite/neat/_session/_inspect.py +22 -10
  76. cognite/neat/_session/_mapping.py +39 -0
  77. cognite/neat/_session/_prepare.py +222 -27
  78. cognite/neat/_session/_read.py +109 -19
  79. cognite/neat/_session/_set.py +2 -2
  80. cognite/neat/_session/_show.py +11 -11
  81. cognite/neat/_session/_to.py +27 -14
  82. cognite/neat/_session/exceptions.py +20 -3
  83. cognite/neat/_store/_base.py +27 -24
  84. cognite/neat/_store/_provenance.py +2 -2
  85. cognite/neat/_utils/auxiliary.py +19 -0
  86. cognite/neat/_utils/rdf_.py +28 -1
  87. cognite/neat/_version.py +1 -1
  88. cognite/neat/_workflows/steps/data_contracts.py +2 -10
  89. cognite/neat/_workflows/steps/lib/current/rules_exporter.py +14 -49
  90. cognite/neat/_workflows/steps/lib/current/rules_importer.py +4 -1
  91. cognite/neat/_workflows/steps/lib/current/rules_validator.py +5 -9
  92. {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/METADATA +4 -3
  93. {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/RECORD +97 -100
  94. cognite/neat/_graph/loaders/_rdf2asset.py +0 -416
  95. cognite/neat/_rules/analysis/_asset.py +0 -173
  96. cognite/neat/_rules/models/asset/__init__.py +0 -13
  97. cognite/neat/_rules/models/asset/_rules.py +0 -109
  98. cognite/neat/_rules/models/asset/_rules_input.py +0 -101
  99. cognite/neat/_rules/models/asset/_validation.py +0 -45
  100. cognite/neat/_rules/models/domain.py +0 -136
  101. cognite/neat/_rules/models/mapping/_base.py +0 -131
  102. cognite/neat/_utils/cdf/loaders/__init__.py +0 -25
  103. cognite/neat/_utils/cdf/loaders/_base.py +0 -54
  104. cognite/neat/_utils/cdf/loaders/_data_modeling.py +0 -339
  105. cognite/neat/_utils/cdf/loaders/_ingestion.py +0 -167
  106. /cognite/neat/{_utils/cdf → _client/_api}/__init__.py +0 -0
  107. {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/LICENSE +0 -0
  108. {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/WHEEL +0 -0
  109. {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/entry_points.txt +0 -0
@@ -2,13 +2,23 @@
2
2
  its sub-models and validators.
3
3
  """
4
4
 
5
- from __future__ import annotations
6
-
5
+ import math
7
6
  import sys
8
7
  import types
9
8
  from abc import ABC, abstractmethod
10
9
  from collections.abc import Callable, Hashable, Iterator, MutableSequence, Sequence
11
- from typing import Annotated, Any, ClassVar, Literal, SupportsIndex, TypeVar, get_args, get_origin, overload
10
+ from datetime import datetime
11
+ from typing import (
12
+ Annotated,
13
+ Any,
14
+ ClassVar,
15
+ Literal,
16
+ SupportsIndex,
17
+ TypeVar,
18
+ get_args,
19
+ get_origin,
20
+ overload,
21
+ )
12
22
 
13
23
  import pandas as pd
14
24
  from pydantic import (
@@ -23,15 +33,25 @@ from pydantic import (
23
33
  )
24
34
  from pydantic.main import IncEx
25
35
  from pydantic_core import core_schema
26
-
27
- from cognite.neat._rules.models._types import ClassEntityType, InformationPropertyType
36
+ from rdflib import Namespace, URIRef
37
+
38
+ from cognite.neat._constants import DEFAULT_NAMESPACE
39
+ from cognite.neat._rules.models._types import (
40
+ ContainerEntityType,
41
+ DataModelExternalIdType,
42
+ DmsPropertyType,
43
+ SpaceType,
44
+ StrListType,
45
+ VersionType,
46
+ ViewEntityType,
47
+ )
48
+ from cognite.neat._rules.models.data_types import DataType
49
+ from cognite.neat._rules.models.entities import EdgeEntity, ReverseConnectionEntity, ViewEntity
28
50
 
29
51
  if sys.version_info >= (3, 11):
30
52
  from enum import StrEnum
31
- from typing import Self
32
53
  else:
33
54
  from backports.strenum import StrEnum
34
- from typing_extensions import Self
35
55
 
36
56
 
37
57
  METADATA_VALUE_MAX_LENGTH = 5120
@@ -82,18 +102,18 @@ class DataModelType(StrEnum):
82
102
  enterprise = "enterprise"
83
103
 
84
104
 
105
+ class DataModelAspect(StrEnum):
106
+ conceptual = "conceptual"
107
+ logical = "logical"
108
+ physical = "physical"
109
+
110
+
85
111
  class RoleTypes(StrEnum):
86
112
  domain_expert = "domain expert"
87
113
  information = "information architect"
88
- asset = "asset architect"
89
114
  dms = "DMS Architect"
90
115
 
91
116
 
92
- class MatchType(StrEnum):
93
- exact = "exact"
94
- partial = "partial"
95
-
96
-
97
117
  class SchemaModel(BaseModel):
98
118
  model_config: ClassVar[ConfigDict] = ConfigDict(
99
119
  populate_by_name=True,
@@ -125,6 +145,46 @@ class BaseMetadata(SchemaModel):
125
145
  """
126
146
 
127
147
  role: ClassVar[RoleTypes]
148
+ aspect: ClassVar[DataModelAspect]
149
+ space: SpaceType = Field(alias="prefix")
150
+ external_id: DataModelExternalIdType = Field(alias="externalId")
151
+ version: VersionType
152
+
153
+ name: str | None = Field(
154
+ None,
155
+ description="Human readable name of the data model",
156
+ min_length=1,
157
+ max_length=255,
158
+ )
159
+
160
+ description: str | None = Field(None, min_length=1, max_length=1024)
161
+
162
+ creator: StrListType = Field(
163
+ description=(
164
+ "List of contributors to the data model creation, "
165
+ "typically information architects are considered as contributors."
166
+ ),
167
+ )
168
+
169
+ created: datetime = Field(
170
+ description=("Date of the data model creation"),
171
+ )
172
+
173
+ updated: datetime = Field(
174
+ description=("Date of the data model update"),
175
+ )
176
+
177
+ @field_validator("*", mode="before")
178
+ def strip_string(cls, value: Any) -> Any:
179
+ if isinstance(value, str):
180
+ return value.strip()
181
+ return value
182
+
183
+ @field_validator("description", mode="before")
184
+ def nan_as_none(cls, value):
185
+ if isinstance(value, float) and math.isnan(value):
186
+ return None
187
+ return value
128
188
 
129
189
  def to_pandas(self) -> pd.Series:
130
190
  """Converts Metadata to pandas Series."""
@@ -143,15 +203,30 @@ class BaseMetadata(SchemaModel):
143
203
  def include_role(self, serializer: Callable) -> dict:
144
204
  return {"role": self.role.value, **serializer(self)}
145
205
 
146
- @abstractmethod
206
+ @property
207
+ def prefix(self) -> str:
208
+ return self.space
209
+
147
210
  def as_identifier(self) -> str:
148
- """Returns a unique identifier for the metadata."""
149
- raise NotImplementedError()
211
+ return f"{self.prefix}:{self.external_id}"
150
212
 
151
- @abstractmethod
152
213
  def get_prefix(self) -> str:
153
- """Returns the prefix for the metadata."""
154
- raise NotImplementedError()
214
+ return self.prefix
215
+
216
+ @property
217
+ def identifier(self) -> URIRef:
218
+ """Globally unique identifier for the data model.
219
+
220
+ !!! note
221
+ Unlike namespace, the identifier does not end with "/" or "#".
222
+
223
+ """
224
+ return DEFAULT_NAMESPACE[f"data-model/verified/{self.aspect}/{self.space}/{self.external_id}/{self.version}"]
225
+
226
+ @property
227
+ def namespace(self) -> Namespace:
228
+ """Namespace for the data model used for the entities in the data model."""
229
+ return Namespace(f"{self.identifier}/")
155
230
 
156
231
 
157
232
  class BaseRules(SchemaModel, ABC):
@@ -169,14 +244,13 @@ class BaseRules(SchemaModel, ABC):
169
244
  """
170
245
 
171
246
  metadata: BaseMetadata
172
- reference: Self | None = Field(None, alias="Reference")
173
247
 
174
248
  @classmethod
175
249
  def headers_by_sheet(cls, by_alias: bool = False) -> dict[str, list[str]]:
176
250
  """Returns a list of headers for the model, typically used by ExcelExporter"""
177
251
  headers_by_sheet: dict[str, list[str]] = {}
178
252
  for field_name, field in cls.model_fields.items():
179
- if field_name == "validators_to_skip":
253
+ if field_name in ["validators_to_skip", "post_validate"]:
180
254
  continue
181
255
  sheet_name = (field.alias or field_name) if by_alias else field_name
182
256
  annotation = field.annotation
@@ -207,7 +281,6 @@ class BaseRules(SchemaModel, ABC):
207
281
  def dump(
208
282
  self,
209
283
  entities_exclude_defaults: bool = True,
210
- as_reference: bool = False,
211
284
  mode: Literal["python", "json"] = "python",
212
285
  by_alias: bool = False,
213
286
  exclude: IncEx | None = None,
@@ -224,9 +297,6 @@ class BaseRules(SchemaModel, ABC):
224
297
  For example, given a class that is dumped as 'my_prefix:MyClass', if the prefix for the rules
225
298
  set in metadata.prefix = 'my_prefix', then this class will be dumped as 'MyClass' when this flag is set.
226
299
  Defaults to True.
227
- as_reference (bool, optional): Whether to dump as reference. For Information and DMS rules, this will
228
- set the reference column/field to the reference of that entity. This is used in the ExcelExporter
229
- to dump a reference model.
230
300
  mode: The mode in which `to_python` should run.
231
301
  If mode is 'json', the output will only contain JSON serializable types.
232
302
  If mode is 'python', the output may contain non-JSON-serializable Python objects.
@@ -242,25 +312,11 @@ class BaseRules(SchemaModel, ABC):
242
312
  if isinstance(value, SheetList):
243
313
  value.sort(key=lambda x: x._identifier())
244
314
 
245
- context: dict[str, Any] = {"as_reference": as_reference}
315
+ context: dict[str, Any] = {}
246
316
  if entities_exclude_defaults:
247
317
  context["metadata"] = self.metadata
248
318
 
249
- exclude_input: IncEx | None
250
- if self.reference is None:
251
- exclude_input = exclude
252
- else:
253
- # If the rules has a reference, we dump that separately with the as_reference flag set to True.
254
- # We don't want to include the reference in the main dump, so we exclude it here.
255
- # This is to include whatever is in the exclude set from the user.
256
- if isinstance(exclude, dict):
257
- exclude_input = exclude.copy()
258
- exclude_input["reference"] = {"__all__"} # type: ignore[index]
259
- elif isinstance(exclude, set):
260
- exclude_input = exclude.copy()
261
- exclude_input.add("reference") # type: ignore[arg-type]
262
- else:
263
- exclude_input = {"reference"}
319
+ exclude_input: IncEx | None = exclude
264
320
 
265
321
  output = self.model_dump(
266
322
  mode=mode,
@@ -271,20 +327,7 @@ class BaseRules(SchemaModel, ABC):
271
327
  exclude_defaults=exclude_defaults,
272
328
  context=context,
273
329
  )
274
- is_reference_user_excluded = isinstance(exclude, dict | set) and "reference" in exclude
275
- if self.reference is not None and not is_reference_user_excluded:
276
- # If the rules has a reference, we dump that separately with the as_reference flag set to True.
277
- # Unless the user has explicitly excluded the reference.
278
- output["Reference" if by_alias else "reference"] = self.reference.dump(
279
- mode=mode,
280
- by_alias=by_alias,
281
- exclude=exclude,
282
- exclude_none=exclude_none,
283
- exclude_unset=exclude_unset,
284
- exclude_defaults=exclude_defaults,
285
- entities_exclude_defaults=entities_exclude_defaults,
286
- as_reference=True,
287
- )
330
+
288
331
  return output
289
332
 
290
333
 
@@ -293,6 +336,10 @@ class SheetRow(SchemaModel):
293
336
  def _identifier(self) -> tuple[Hashable, ...]:
294
337
  raise NotImplementedError()
295
338
 
339
+ def __repr__(self) -> str:
340
+ # Simplified representation of the object for debugging
341
+ return f"{self.__class__.__name__}({self._identifier()})"
342
+
296
343
 
297
344
  T_SheetRow = TypeVar("T_SheetRow", bound=SheetRow)
298
345
 
@@ -333,9 +380,9 @@ class SheetList(list, MutableSequence[T_SheetRow]):
333
380
  def __getitem__(self, index: SupportsIndex) -> T_SheetRow: ...
334
381
 
335
382
  @overload
336
- def __getitem__(self, index: slice) -> SheetList[T_SheetRow]: ...
383
+ def __getitem__(self, index: slice) -> "SheetList[T_SheetRow]": ...
337
384
 
338
- def __getitem__(self, index: SupportsIndex | slice, /) -> T_SheetRow | SheetList[T_SheetRow]:
385
+ def __getitem__(self, index: SupportsIndex | slice, /) -> "T_SheetRow | SheetList[T_SheetRow]":
339
386
  if isinstance(index, slice):
340
387
  return SheetList[T_SheetRow](super().__getitem__(index))
341
388
  return super().__getitem__(index)
@@ -353,10 +400,19 @@ ExtensionCategoryType = Annotated[
353
400
 
354
401
 
355
402
  # Immutable such that this can be used as a key in a dictionary
356
- class PropertyRef(BaseModel, frozen=True):
357
- class_: ClassEntityType = Field(alias="Class")
358
- property_: InformationPropertyType = Field(alias="Property")
403
+ class ContainerProperty(BaseModel, frozen=True):
404
+ container: ContainerEntityType
405
+ property_: DmsPropertyType
406
+
407
+
408
+ class ContainerDestinationProperty(ContainerProperty, frozen=True):
409
+ value_type: DataType | ViewEntity
410
+ connection: Literal["direct"] | ReverseConnectionEntity | EdgeEntity | None = None
411
+
412
+
413
+ class ViewRef(BaseModel, frozen=True):
414
+ view: ViewEntityType
359
415
 
360
416
 
361
- class ClassRef(BaseModel, frozen=True):
362
- class_: ClassEntityType = Field(alias="Class")
417
+ class ViewProperty(ViewRef, frozen=True):
418
+ property_: DmsPropertyType
@@ -1,3 +1,5 @@
1
+ from cognite.neat._client.data_classes.schema import DMSSchema
2
+
1
3
  from ._rules import DMSContainer, DMSEnum, DMSMetadata, DMSNode, DMSProperty, DMSRules, DMSView
2
4
  from ._rules_input import (
3
5
  DMSInputContainer,
@@ -8,7 +10,6 @@ from ._rules_input import (
8
10
  DMSInputRules,
9
11
  DMSInputView,
10
12
  )
11
- from ._schema import DMSSchema, PipelineSchema
12
13
 
13
14
  __all__ = [
14
15
  "DMSRules",
@@ -19,7 +20,6 @@ __all__ = [
19
20
  "DMSContainer",
20
21
  "DMSNode",
21
22
  "DMSEnum",
22
- "PipelineSchema",
23
23
  "DMSInputRules",
24
24
  "DMSInputMetadata",
25
25
  "DMSInputView",
@@ -1,7 +1,7 @@
1
1
  import warnings
2
2
  from collections import defaultdict
3
3
  from collections.abc import Collection, Hashable, Sequence
4
- from typing import Any, cast
4
+ from typing import Any
5
5
 
6
6
  from cognite.client.data_classes import data_modeling as dm
7
7
  from cognite.client.data_classes.data_modeling.containers import BTreeIndex
@@ -13,15 +13,19 @@ from cognite.client.data_classes.data_modeling.views import (
13
13
  ViewPropertyApply,
14
14
  )
15
15
 
16
+ from cognite.neat._client.data_classes.data_modeling import (
17
+ ContainerApplyDict,
18
+ NodeApplyDict,
19
+ SpaceApplyDict,
20
+ ViewApplyDict,
21
+ )
22
+ from cognite.neat._client.data_classes.schema import DMSSchema
16
23
  from cognite.neat._issues.errors import NeatTypeError, ResourceNotFoundError
17
24
  from cognite.neat._issues.warnings import NotSupportedWarning, PropertyNotFoundWarning
18
25
  from cognite.neat._issues.warnings.user_modeling import (
19
26
  EmptyContainerWarning,
20
27
  HasDataFilterOnNoPropertiesViewWarning,
21
- HasDataFilterOnViewWithReferencesWarning,
22
- NodeTypeFilterOnParentViewWarning,
23
28
  )
24
- from cognite.neat._rules.models._base_rules import DataModelType, ExtensionCategory, SchemaCompleteness
25
29
  from cognite.neat._rules.models.data_types import DataType, Double, Enum, Float
26
30
  from cognite.neat._rules.models.entities import (
27
31
  ClassEntity,
@@ -32,15 +36,12 @@ from cognite.neat._rules.models.entities import (
32
36
  EdgeEntity,
33
37
  HasDataFilter,
34
38
  NodeTypeFilter,
35
- ReferenceEntity,
36
39
  ReverseConnectionEntity,
37
40
  UnitEntity,
38
41
  ViewEntity,
39
42
  )
40
- from cognite.neat._utils.cdf.data_classes import ContainerApplyDict, NodeApplyDict, SpaceApplyDict, ViewApplyDict
41
43
 
42
44
  from ._rules import DMSEnum, DMSMetadata, DMSProperty, DMSRules, DMSView
43
- from ._schema import DMSSchema, PipelineSchema
44
45
 
45
46
 
46
47
  class _DMSExporter:
@@ -55,78 +56,33 @@ class _DMSExporter:
55
56
  instance_space (str): The space to use for the instance. Defaults to None,`Rules.metadata.space` will be used
56
57
  """
57
58
 
58
- def __init__(
59
- self,
60
- rules: DMSRules,
61
- include_pipeline: bool = False,
62
- instance_space: str | None = None,
63
- ):
64
- self.include_pipeline = include_pipeline
59
+ def __init__(self, rules: DMSRules, instance_space: str | None = None):
65
60
  self.instance_space = instance_space
66
61
  self.rules = rules
67
- self._ref_schema = rules.reference.as_schema() if rules.reference else None
62
+ self._ref_schema = None
68
63
  if self._ref_schema:
69
64
  # We skip version as that will always be missing in the reference
70
65
  self._ref_views_by_id = {
71
66
  dm.ViewId(view.space, view.external_id): view for view in self._ref_schema.views.values()
72
67
  }
73
68
  else:
74
- self._ref_views_by_id = {}
75
-
76
- self.is_addition = (
77
- rules.metadata.schema_ is SchemaCompleteness.extended
78
- and rules.metadata.extension is ExtensionCategory.addition
79
- )
80
- self.is_reshape = (
81
- rules.metadata.schema_ is SchemaCompleteness.extended
82
- and rules.metadata.extension is ExtensionCategory.reshape
83
- )
84
- self.is_rebuild = (
85
- rules.metadata.schema_ is SchemaCompleteness.extended
86
- and rules.metadata.extension is ExtensionCategory.rebuild
87
- )
69
+ self._ref_views_by_id = {} # type: ignore
88
70
 
89
71
  def to_schema(self) -> DMSSchema:
90
72
  rules = self.rules
91
73
  container_properties_by_id, view_properties_by_id = self._gather_properties(list(self.rules.properties))
92
74
 
93
- # If we are reshaping or rebuilding, and there are no properties in the current rules, we will
94
- # include those properties from the last rules.
95
- if rules.last and (self.is_reshape or self.is_rebuild):
96
- selected_views = {view.view for view in rules.views}
97
- selected_properties = [
98
- prop
99
- for prop in rules.last.properties
100
- if prop.view in selected_views and prop.view.as_id() not in view_properties_by_id
101
- ]
102
- self._update_with_properties(
103
- selected_properties, container_properties_by_id, view_properties_by_id, include_new_containers=True
104
- )
105
-
106
- # We need to include the properties from the last rules as well to create complete containers and view
107
- # depending on the type of extension.
108
- if rules.last and self.is_addition:
109
- selected_properties = [
110
- prop for prop in rules.last.properties if (prop.view.as_id() in view_properties_by_id)
111
- ]
112
- self._update_with_properties(selected_properties, container_properties_by_id, view_properties_by_id)
113
- elif rules.last and (self.is_reshape or self.is_rebuild):
114
- selected_properties = [
115
- prop
116
- for prop in rules.last.properties
117
- if prop.container and prop.container.as_id() in container_properties_by_id
118
- ]
119
- self._update_with_properties(selected_properties, container_properties_by_id, None)
120
-
121
75
  containers = self._create_containers(container_properties_by_id, rules.enum) # type: ignore[arg-type]
122
76
 
123
77
  view_properties_with_ancestors_by_id = self._gather_properties_with_ancestors(
124
78
  view_properties_by_id, rules.views
125
79
  )
126
80
 
127
- views, view_node_type_filters = self._create_views_with_node_types(
128
- view_properties_by_id, view_properties_with_ancestors_by_id
129
- )
81
+ views = self._create_views_with_node_types(view_properties_by_id, view_properties_with_ancestors_by_id)
82
+ view_node_type_filters: set[dm.NodeId] = set()
83
+ for dms_view in rules.views:
84
+ if isinstance(dms_view.filter_, NodeTypeFilter):
85
+ view_node_type_filters.update(node.as_id() for node in dms_view.filter_.inner or [])
130
86
  if rules.nodes:
131
87
  node_types = NodeApplyDict(
132
88
  [node.as_node() for node in rules.nodes]
@@ -136,30 +92,11 @@ class _DMSExporter:
136
92
  node_types = NodeApplyDict([dm.NodeApply(node.space, node.external_id) for node in view_node_type_filters])
137
93
 
138
94
  last_schema: DMSSchema | None = None
139
- if self.rules.last:
140
- last_schema = self.rules.last.as_schema()
141
- # Remove the views that are in the current model, last + current should equal the full model
142
- # without any duplicates
143
- last_schema.views = ViewApplyDict(
144
- {view_id: view for view_id, view in last_schema.views.items() if view_id not in views}
145
- )
146
- last_schema.containers = ContainerApplyDict(
147
- {
148
- container_id: container
149
- for container_id, container in last_schema.containers.items()
150
- if container_id not in containers
151
- }
152
- )
153
95
 
154
96
  views_not_in_model = {view.view.as_id() for view in rules.views if not view.in_model}
155
- if rules.last and self.is_addition:
156
- views_not_in_model.update({view.view.as_id() for view in rules.last.views if not view.in_model})
157
-
158
97
  data_model = rules.metadata.as_data_model()
159
98
 
160
99
  data_model_views = [view_id for view_id in views if view_id not in views_not_in_model]
161
- if last_schema and self.is_addition:
162
- data_model_views.extend([view_id for view_id in last_schema.views if view_id not in views_not_in_model])
163
100
 
164
101
  # Sorting to ensure deterministic order
165
102
  data_model.views = sorted(data_model_views, key=lambda v: v.as_tuple()) # type: ignore[union-attr]
@@ -173,8 +110,6 @@ class _DMSExporter:
173
110
  containers=containers,
174
111
  node_types=node_types,
175
112
  )
176
- if self.include_pipeline:
177
- return PipelineSchema.from_dms(output, self.instance_space)
178
113
 
179
114
  if self._ref_schema:
180
115
  output.reference = self._ref_schema
@@ -205,19 +140,10 @@ class _DMSExporter:
205
140
  self,
206
141
  view_properties_by_id: dict[dm.ViewId, list[DMSProperty]],
207
142
  view_properties_with_ancestors_by_id: dict[dm.ViewId, list[DMSProperty]],
208
- ) -> tuple[ViewApplyDict, set[dm.NodeId]]:
143
+ ) -> ViewApplyDict:
209
144
  input_views = list(self.rules.views)
210
- if self.rules.last:
211
- existing = {view.view.as_id() for view in input_views}
212
- modified_views = [
213
- v
214
- for v in self.rules.last.views
215
- if v.view.as_id() in view_properties_by_id and v.view.as_id() not in existing
216
- ]
217
- input_views.extend(modified_views)
218
145
 
219
146
  views = ViewApplyDict([dms_view.as_view() for dms_view in input_views])
220
- dms_view_by_id = {dms_view.view.as_id(): dms_view for dms_view in input_views}
221
147
 
222
148
  for view_id, view in views.items():
223
149
  view.properties = {}
@@ -228,53 +154,12 @@ class _DMSExporter:
228
154
  if view_property is not None:
229
155
  view.properties[prop.view_property] = view_property
230
156
 
231
- data_model_type = self.rules.metadata.data_model_type
232
- unique_node_types: set[dm.NodeId] = set()
233
- parent_views = {parent for view in views.values() for parent in view.implements or []}
234
- for view_id, view in views.items():
235
- dms_view = dms_view_by_id.get(view_id)
236
- dms_properties = view_properties_by_id.get(view_id, [])
237
- view_filter = self._create_view_filter(view, dms_view, data_model_type, dms_properties)
238
-
239
- view.filter = view_filter.as_dms_filter()
240
- if isinstance(view_filter, NodeTypeFilter):
241
- unique_node_types.update(view_filter.nodes)
242
- if view.as_id() in parent_views:
243
- warnings.warn(
244
- NodeTypeFilterOnParentViewWarning(view.as_id()),
245
- stacklevel=2,
246
- )
247
-
248
- elif isinstance(view_filter, HasDataFilter) and data_model_type == DataModelType.solution:
249
- if dms_view and isinstance(dms_view.reference, ReferenceEntity):
250
- references = {dms_view.reference.as_view_id()}
251
- elif any(True for prop in dms_properties if isinstance(prop.reference, ReferenceEntity)):
252
- references = {
253
- prop.reference.as_view_id()
254
- for prop in dms_properties
255
- if isinstance(prop.reference, ReferenceEntity)
256
- }
257
- else:
258
- continue
259
- warnings.warn(
260
- HasDataFilterOnViewWithReferencesWarning(view.as_id(), frozenset(references)),
261
- stacklevel=2,
262
- )
263
-
264
- if data_model_type == DataModelType.enterprise:
265
- # Enterprise Model needs to create node types for all views,
266
- # as they are expected for the solution model.
267
- unique_node_types.add(dm.NodeId(space=view.space, external_id=view.external_id))
268
-
269
- return views, unique_node_types
157
+ return views
270
158
 
271
159
  @classmethod
272
160
  def _create_edge_type_from_prop(cls, prop: DMSProperty) -> dm.DirectRelationReference:
273
161
  if isinstance(prop.connection, EdgeEntity) and prop.connection.edge_type is not None:
274
162
  return prop.connection.edge_type.as_reference()
275
- elif isinstance(prop.reference, ReferenceEntity):
276
- ref_view_prop = prop.reference.as_view_property_id()
277
- return cls._create_edge_type_from_view_id(cast(dm.ViewId, ref_view_prop.source), ref_view_prop.property)
278
163
  elif isinstance(prop.value_type, ViewEntity):
279
164
  return cls._create_edge_type_from_view_id(prop.view.as_id(), prop.view_property)
280
165
  else:
@@ -298,14 +183,6 @@ class _DMSExporter:
298
183
  enum_values_by_collection[enum_value.collection].append(enum_value)
299
184
 
300
185
  containers = list(self.rules.containers or [])
301
- if self.rules.last:
302
- existing = {container.container.as_id() for container in containers}
303
- modified_containers = [
304
- c
305
- for c in self.rules.last.containers or []
306
- if c.container.as_id() in container_properties_by_id and c.container.as_id() not in existing
307
- ]
308
- containers.extend(modified_containers)
309
186
 
310
187
  containers = dm.ContainerApplyList([dms_container.as_container() for dms_container in containers])
311
188
  container_to_drop = set()
@@ -408,17 +285,6 @@ class _DMSExporter:
408
285
  views: Sequence[DMSView],
409
286
  ) -> dict[dm.ViewId, list[DMSProperty]]:
410
287
  all_view_properties_by_id = view_properties_by_id.copy()
411
- if self.rules.reference:
412
- # We need to include t
413
- ref_view_properties_by_id = self._gather_properties(self.rules.reference.properties)[1]
414
- for view_id, properties in ref_view_properties_by_id.items():
415
- if view_id not in all_view_properties_by_id:
416
- all_view_properties_by_id[view_id] = properties
417
- else:
418
- existing_properties = {prop._identifier() for prop in all_view_properties_by_id[view_id]}
419
- for prop in properties:
420
- if prop._identifier() not in existing_properties:
421
- all_view_properties_by_id[view_id].append(prop)
422
288
 
423
289
  view_properties_with_parents_by_id: dict[dm.ViewId, list[DMSProperty]] = defaultdict(list)
424
290
  view_by_view_id = {view.view.as_id(): view for view in views}
@@ -482,37 +348,13 @@ class _DMSExporter:
482
348
  self,
483
349
  view: dm.ViewApply,
484
350
  dms_view: DMSView | None,
485
- data_model_type: DataModelType,
486
- dms_properties: list[DMSProperty],
487
- ) -> DMSFilter:
351
+ ) -> DMSFilter | None:
488
352
  selected_filter_name = (dms_view and dms_view.filter_ and dms_view.filter_.name) or ""
489
353
 
490
354
  if dms_view and dms_view.filter_ and not dms_view.filter_.is_empty:
491
355
  # Has Explicit Filter, use it
492
356
  return dms_view.filter_
493
357
 
494
- if data_model_type == DataModelType.solution and selected_filter_name in [NodeTypeFilter.name, ""]:
495
- if (
496
- dms_view
497
- and isinstance(dms_view.reference, ReferenceEntity)
498
- and not dms_properties
499
- and (ref_view := self._ref_views_by_id.get(dms_view.reference.as_view_id()))
500
- and ref_view.filter
501
- ):
502
- # No new properties, only reference, reuse the reference filter
503
- return DMSFilter.from_dms_filter(ref_view.filter)
504
- else:
505
- referenced_node_ids = {
506
- prop.reference.as_node_entity()
507
- for prop in dms_properties
508
- if isinstance(prop.reference, ReferenceEntity)
509
- }
510
- if dms_view and isinstance(dms_view.reference, ReferenceEntity):
511
- referenced_node_ids.add(dms_view.reference.as_node_entity())
512
-
513
- if referenced_node_ids:
514
- return NodeTypeFilter(inner=list(referenced_node_ids))
515
-
516
358
  # Enterprise Model or (Solution + HasData)
517
359
  ref_containers = view.referenced_containers()
518
360
  if not ref_containers or selected_filter_name == HasDataFilter.name:
@@ -597,7 +439,7 @@ class _DMSExporter:
597
439
  (
598
440
  prop
599
441
  for prop in view_properties_with_ancestors_by_id.get(source_view_id, [])
600
- if prop.property_ == reverse_prop_id
442
+ if prop.view_property == reverse_prop_id
601
443
  ),
602
444
  None,
603
445
  )