cognite-neat 0.103.1__py3-none-any.whl → 0.105.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cognite-neat might be problematic. Click here for more details.

Files changed (175) hide show
  1. cognite/neat/_client/_api/data_modeling_loaders.py +83 -23
  2. cognite/neat/_client/_api/schema.py +2 -1
  3. cognite/neat/_client/data_classes/neat_sequence.py +261 -0
  4. cognite/neat/_client/data_classes/schema.py +5 -1
  5. cognite/neat/_client/testing.py +33 -0
  6. cognite/neat/_constants.py +56 -0
  7. cognite/neat/_graph/extractors/_classic_cdf/_base.py +6 -5
  8. cognite/neat/_graph/extractors/_classic_cdf/_sequences.py +225 -11
  9. cognite/neat/_graph/extractors/_mock_graph_generator.py +2 -2
  10. cognite/neat/_graph/loaders/_rdf2dms.py +13 -2
  11. cognite/neat/_graph/transformers/__init__.py +3 -1
  12. cognite/neat/_graph/transformers/_base.py +109 -1
  13. cognite/neat/_graph/transformers/_classic_cdf.py +6 -1
  14. cognite/neat/_graph/transformers/_prune_graph.py +103 -47
  15. cognite/neat/_graph/transformers/_rdfpath.py +41 -17
  16. cognite/neat/_graph/transformers/_value_type.py +188 -151
  17. cognite/neat/_issues/__init__.py +0 -2
  18. cognite/neat/_issues/_base.py +54 -43
  19. cognite/neat/_issues/warnings/__init__.py +4 -1
  20. cognite/neat/_issues/warnings/_general.py +7 -0
  21. cognite/neat/_issues/warnings/_resources.py +12 -1
  22. cognite/neat/_rules/_shared.py +18 -34
  23. cognite/neat/_rules/exporters/_base.py +28 -2
  24. cognite/neat/_rules/exporters/_rules2dms.py +39 -1
  25. cognite/neat/_rules/exporters/_rules2excel.py +13 -2
  26. cognite/neat/_rules/exporters/_rules2instance_template.py +4 -0
  27. cognite/neat/_rules/exporters/_rules2ontology.py +13 -1
  28. cognite/neat/_rules/exporters/_rules2yaml.py +4 -0
  29. cognite/neat/_rules/importers/_base.py +9 -0
  30. cognite/neat/_rules/importers/_dms2rules.py +80 -57
  31. cognite/neat/_rules/importers/_dtdl2rules/dtdl_importer.py +5 -2
  32. cognite/neat/_rules/importers/_rdf/_base.py +10 -8
  33. cognite/neat/_rules/importers/_rdf/_imf2rules.py +4 -0
  34. cognite/neat/_rules/importers/_rdf/_inference2rules.py +7 -0
  35. cognite/neat/_rules/importers/_rdf/_owl2rules.py +4 -0
  36. cognite/neat/_rules/importers/_spreadsheet2rules.py +17 -8
  37. cognite/neat/_rules/importers/_yaml2rules.py +21 -7
  38. cognite/neat/_rules/models/_base_input.py +1 -1
  39. cognite/neat/_rules/models/_base_rules.py +9 -1
  40. cognite/neat/_rules/models/dms/_rules.py +4 -0
  41. cognite/neat/_rules/models/dms/_rules_input.py +9 -0
  42. cognite/neat/_rules/models/entities/_wrapped.py +10 -5
  43. cognite/neat/_rules/models/information/_rules.py +4 -0
  44. cognite/neat/_rules/models/information/_rules_input.py +9 -0
  45. cognite/neat/_rules/models/mapping/_classic2core.py +2 -5
  46. cognite/neat/_rules/models/mapping/_classic2core.yaml +239 -38
  47. cognite/neat/_rules/transformers/__init__.py +13 -6
  48. cognite/neat/_rules/transformers/_base.py +41 -65
  49. cognite/neat/_rules/transformers/_converters.py +404 -234
  50. cognite/neat/_rules/transformers/_mapping.py +93 -72
  51. cognite/neat/_rules/transformers/_verification.py +50 -38
  52. cognite/neat/_session/_base.py +32 -121
  53. cognite/neat/_session/_inspect.py +5 -3
  54. cognite/neat/_session/_mapping.py +17 -105
  55. cognite/neat/_session/_prepare.py +138 -268
  56. cognite/neat/_session/_read.py +39 -195
  57. cognite/neat/_session/_set.py +6 -30
  58. cognite/neat/_session/_show.py +40 -21
  59. cognite/neat/_session/_state.py +49 -107
  60. cognite/neat/_session/_to.py +44 -33
  61. cognite/neat/_shared.py +23 -2
  62. cognite/neat/_store/_provenance.py +3 -82
  63. cognite/neat/_store/_rules_store.py +368 -10
  64. cognite/neat/_store/exceptions.py +23 -0
  65. cognite/neat/_utils/graph_transformations_report.py +36 -0
  66. cognite/neat/_utils/rdf_.py +8 -0
  67. cognite/neat/_utils/reader/_base.py +27 -0
  68. cognite/neat/_utils/spreadsheet.py +5 -4
  69. cognite/neat/_version.py +1 -1
  70. {cognite_neat-0.103.1.dist-info → cognite_neat-0.105.0.dist-info}/METADATA +3 -2
  71. cognite_neat-0.105.0.dist-info/RECORD +179 -0
  72. {cognite_neat-0.103.1.dist-info → cognite_neat-0.105.0.dist-info}/WHEEL +1 -1
  73. cognite/neat/_app/api/__init__.py +0 -0
  74. cognite/neat/_app/api/asgi/metrics.py +0 -4
  75. cognite/neat/_app/api/configuration.py +0 -98
  76. cognite/neat/_app/api/context_manager/__init__.py +0 -3
  77. cognite/neat/_app/api/context_manager/manager.py +0 -16
  78. cognite/neat/_app/api/data_classes/__init__.py +0 -0
  79. cognite/neat/_app/api/data_classes/rest.py +0 -59
  80. cognite/neat/_app/api/explorer.py +0 -66
  81. cognite/neat/_app/api/routers/configuration.py +0 -25
  82. cognite/neat/_app/api/routers/crud.py +0 -102
  83. cognite/neat/_app/api/routers/metrics.py +0 -10
  84. cognite/neat/_app/api/routers/workflows.py +0 -224
  85. cognite/neat/_app/api/utils/__init__.py +0 -0
  86. cognite/neat/_app/api/utils/data_mapping.py +0 -17
  87. cognite/neat/_app/api/utils/logging.py +0 -26
  88. cognite/neat/_app/api/utils/query_templates.py +0 -92
  89. cognite/neat/_app/main.py +0 -17
  90. cognite/neat/_app/monitoring/__init__.py +0 -0
  91. cognite/neat/_app/monitoring/metrics.py +0 -69
  92. cognite/neat/_app/ui/index.html +0 -1
  93. cognite/neat/_app/ui/neat-app/.gitignore +0 -23
  94. cognite/neat/_app/ui/neat-app/README.md +0 -70
  95. cognite/neat/_app/ui/neat-app/build/asset-manifest.json +0 -14
  96. cognite/neat/_app/ui/neat-app/build/favicon.ico +0 -0
  97. cognite/neat/_app/ui/neat-app/build/img/architect-icon.svg +0 -116
  98. cognite/neat/_app/ui/neat-app/build/img/developer-icon.svg +0 -112
  99. cognite/neat/_app/ui/neat-app/build/img/sme-icon.svg +0 -34
  100. cognite/neat/_app/ui/neat-app/build/index.html +0 -1
  101. cognite/neat/_app/ui/neat-app/build/logo192.png +0 -0
  102. cognite/neat/_app/ui/neat-app/build/manifest.json +0 -25
  103. cognite/neat/_app/ui/neat-app/build/robots.txt +0 -3
  104. cognite/neat/_app/ui/neat-app/build/static/css/main.72e3d92e.css +0 -2
  105. cognite/neat/_app/ui/neat-app/build/static/css/main.72e3d92e.css.map +0 -1
  106. cognite/neat/_app/ui/neat-app/build/static/js/main.5a52cf09.js +0 -3
  107. cognite/neat/_app/ui/neat-app/build/static/js/main.5a52cf09.js.LICENSE.txt +0 -88
  108. cognite/neat/_app/ui/neat-app/build/static/js/main.5a52cf09.js.map +0 -1
  109. cognite/neat/_app/ui/neat-app/build/static/media/logo.8093b84df9ed36a174c629d6fe0b730d.svg +0 -1
  110. cognite/neat/_app/ui/neat-app/package-lock.json +0 -18306
  111. cognite/neat/_app/ui/neat-app/package.json +0 -62
  112. cognite/neat/_app/ui/neat-app/public/favicon.ico +0 -0
  113. cognite/neat/_app/ui/neat-app/public/img/architect-icon.svg +0 -116
  114. cognite/neat/_app/ui/neat-app/public/img/developer-icon.svg +0 -112
  115. cognite/neat/_app/ui/neat-app/public/img/sme-icon.svg +0 -34
  116. cognite/neat/_app/ui/neat-app/public/index.html +0 -43
  117. cognite/neat/_app/ui/neat-app/public/logo192.png +0 -0
  118. cognite/neat/_app/ui/neat-app/public/manifest.json +0 -25
  119. cognite/neat/_app/ui/neat-app/public/robots.txt +0 -3
  120. cognite/neat/_app/ui/neat-app/src/App.css +0 -38
  121. cognite/neat/_app/ui/neat-app/src/App.js +0 -17
  122. cognite/neat/_app/ui/neat-app/src/App.test.js +0 -8
  123. cognite/neat/_app/ui/neat-app/src/MainContainer.tsx +0 -70
  124. cognite/neat/_app/ui/neat-app/src/components/JsonViewer.tsx +0 -43
  125. cognite/neat/_app/ui/neat-app/src/components/LocalUploader.tsx +0 -124
  126. cognite/neat/_app/ui/neat-app/src/components/OverviewComponentEditorDialog.tsx +0 -63
  127. cognite/neat/_app/ui/neat-app/src/components/StepEditorDialog.tsx +0 -511
  128. cognite/neat/_app/ui/neat-app/src/components/TabPanel.tsx +0 -36
  129. cognite/neat/_app/ui/neat-app/src/components/Utils.tsx +0 -56
  130. cognite/neat/_app/ui/neat-app/src/components/WorkflowDeleteDialog.tsx +0 -60
  131. cognite/neat/_app/ui/neat-app/src/components/WorkflowExecutionReport.tsx +0 -112
  132. cognite/neat/_app/ui/neat-app/src/components/WorkflowImportExportDialog.tsx +0 -67
  133. cognite/neat/_app/ui/neat-app/src/components/WorkflowMetadataDialog.tsx +0 -79
  134. cognite/neat/_app/ui/neat-app/src/index.css +0 -13
  135. cognite/neat/_app/ui/neat-app/src/index.js +0 -13
  136. cognite/neat/_app/ui/neat-app/src/logo.svg +0 -1
  137. cognite/neat/_app/ui/neat-app/src/reportWebVitals.js +0 -13
  138. cognite/neat/_app/ui/neat-app/src/setupTests.js +0 -5
  139. cognite/neat/_app/ui/neat-app/src/types/WorkflowTypes.ts +0 -388
  140. cognite/neat/_app/ui/neat-app/src/views/AboutView.tsx +0 -61
  141. cognite/neat/_app/ui/neat-app/src/views/ConfigView.tsx +0 -184
  142. cognite/neat/_app/ui/neat-app/src/views/GlobalConfigView.tsx +0 -180
  143. cognite/neat/_app/ui/neat-app/src/views/WorkflowView.tsx +0 -570
  144. cognite/neat/_app/ui/neat-app/tsconfig.json +0 -27
  145. cognite/neat/_rules/transformers/_pipelines.py +0 -70
  146. cognite/neat/_workflows/__init__.py +0 -17
  147. cognite/neat/_workflows/base.py +0 -590
  148. cognite/neat/_workflows/cdf_store.py +0 -393
  149. cognite/neat/_workflows/examples/Export_DMS/workflow.yaml +0 -89
  150. cognite/neat/_workflows/examples/Export_Semantic_Data_Model/workflow.yaml +0 -66
  151. cognite/neat/_workflows/examples/Import_DMS/workflow.yaml +0 -65
  152. cognite/neat/_workflows/examples/Validate_Rules/workflow.yaml +0 -67
  153. cognite/neat/_workflows/examples/Validate_Solution_Model/workflow.yaml +0 -64
  154. cognite/neat/_workflows/manager.py +0 -292
  155. cognite/neat/_workflows/model.py +0 -203
  156. cognite/neat/_workflows/steps/__init__.py +0 -0
  157. cognite/neat/_workflows/steps/data_contracts.py +0 -109
  158. cognite/neat/_workflows/steps/lib/__init__.py +0 -0
  159. cognite/neat/_workflows/steps/lib/current/__init__.py +0 -6
  160. cognite/neat/_workflows/steps/lib/current/graph_extractor.py +0 -100
  161. cognite/neat/_workflows/steps/lib/current/graph_loader.py +0 -51
  162. cognite/neat/_workflows/steps/lib/current/graph_store.py +0 -48
  163. cognite/neat/_workflows/steps/lib/current/rules_exporter.py +0 -537
  164. cognite/neat/_workflows/steps/lib/current/rules_importer.py +0 -398
  165. cognite/neat/_workflows/steps/lib/current/rules_validator.py +0 -106
  166. cognite/neat/_workflows/steps/lib/io/__init__.py +0 -1
  167. cognite/neat/_workflows/steps/lib/io/io_steps.py +0 -393
  168. cognite/neat/_workflows/steps/step_model.py +0 -79
  169. cognite/neat/_workflows/steps_registry.py +0 -218
  170. cognite/neat/_workflows/tasks.py +0 -18
  171. cognite/neat/_workflows/triggers.py +0 -169
  172. cognite/neat/_workflows/utils.py +0 -19
  173. cognite_neat-0.103.1.dist-info/RECORD +0 -275
  174. {cognite_neat-0.103.1.dist-info → cognite_neat-0.105.0.dist-info}/LICENSE +0 -0
  175. {cognite_neat-0.103.1.dist-info → cognite_neat-0.105.0.dist-info}/entry_points.txt +0 -0
@@ -1,8 +1,6 @@
1
- from abc import ABC, abstractmethod
2
1
  from dataclasses import dataclass
3
2
  from typing import Any, Generic, TypeAlias, TypeVar
4
3
 
5
- from cognite.neat._issues import IssueList
6
4
  from cognite.neat._rules.models import (
7
5
  DMSRules,
8
6
  InformationRules,
@@ -11,46 +9,32 @@ from cognite.neat._rules.models.dms._rules_input import DMSInputRules
11
9
  from cognite.neat._rules.models.information._rules_input import InformationInputRules
12
10
 
13
11
  VerifiedRules: TypeAlias = InformationRules | DMSRules
14
- InputRules: TypeAlias = DMSInputRules | InformationInputRules
15
- Rules: TypeAlias = DMSInputRules | InformationInputRules | InformationRules | DMSRules
16
- T_Rules = TypeVar("T_Rules", bound=Rules)
12
+
17
13
  T_VerifiedRules = TypeVar("T_VerifiedRules", bound=VerifiedRules)
14
+ InputRules: TypeAlias = DMSInputRules | InformationInputRules
18
15
  T_InputRules = TypeVar("T_InputRules", bound=InputRules)
19
16
 
20
17
 
21
18
  @dataclass
22
- class OutRules(Generic[T_Rules], ABC):
23
- """This is a base class for all rule states."""
24
-
25
- @abstractmethod
26
- def get_rules(self) -> T_Rules | None:
27
- """Get the rules from the state."""
28
- raise NotImplementedError()
29
-
30
-
31
- @dataclass
32
- class JustRules(OutRules[T_Rules]):
33
- """This represents a rule that exists"""
19
+ class ReadRules(Generic[T_InputRules]):
20
+ """This represents a rules that has been read."""
34
21
 
35
- rules: T_Rules
36
-
37
- def get_rules(self) -> T_Rules:
38
- return self.rules
39
-
40
-
41
- @dataclass
42
- class MaybeRules(OutRules[T_Rules]):
43
- """This represents a rule that may or may not exist"""
22
+ rules: T_InputRules | None
23
+ read_context: dict[str, Any]
44
24
 
45
- rules: T_Rules | None
46
- issues: IssueList
25
+ @classmethod
26
+ def display_type_name(cls) -> str:
27
+ return "UnverifiedModel"
47
28
 
48
- def get_rules(self) -> T_Rules | None:
49
- return self.rules
29
+ @property
30
+ def display_name(self):
31
+ if self.rules is None:
32
+ return "FailedRead"
33
+ return self.rules.display_name
50
34
 
51
35
 
52
- @dataclass
53
- class ReadRules(MaybeRules[T_Rules]):
54
- """This represents a rule that does not exist"""
36
+ ReadInputRules: TypeAlias = ReadRules[DMSInputRules] | ReadRules[InformationInputRules]
37
+ T_ReadInputRules = TypeVar("T_ReadInputRules", bound=ReadInputRules)
55
38
 
56
- read_context: dict[str, Any]
39
+ Rules: TypeAlias = InformationRules | DMSRules | ReadRules[DMSInputRules] | ReadRules[InformationInputRules]
40
+ T_Rules = TypeVar("T_Rules", bound=Rules)
@@ -1,13 +1,19 @@
1
1
  from abc import ABC, abstractmethod
2
2
  from collections.abc import Iterable
3
+ from functools import lru_cache
3
4
  from pathlib import Path
4
- from typing import Generic, TypeVar
5
+ from types import UnionType
6
+ from typing import TYPE_CHECKING, Generic, TypeVar, Union, get_args, get_origin
5
7
 
6
8
  from cognite.neat._client import NeatClient
9
+ from cognite.neat._constants import DEFAULT_NAMESPACE
7
10
  from cognite.neat._rules._shared import T_VerifiedRules
8
11
  from cognite.neat._utils.auxiliary import class_html_doc
9
12
  from cognite.neat._utils.upload import UploadResult, UploadResultList
10
13
 
14
+ if TYPE_CHECKING:
15
+ from cognite.neat._store._provenance import Agent as ProvenanceAgent
16
+
11
17
  T_Export = TypeVar("T_Export")
12
18
 
13
19
 
@@ -27,8 +33,28 @@ class BaseExporter(ABC, Generic[T_VerifiedRules, T_Export]):
27
33
  def _repr_html_(cls) -> str:
28
34
  return class_html_doc(cls, include_factory_methods=False)
29
35
 
36
+ @property
37
+ def agent(self) -> "ProvenanceAgent":
38
+ """Provenance agent for the importer."""
39
+ from cognite.neat._store._provenance import Agent as ProvenanceAgent
40
+
41
+ return ProvenanceAgent(id_=DEFAULT_NAMESPACE[f"agent/{type(self).__name__}"])
42
+
43
+ @property
44
+ def description(self) -> str:
45
+ return "MISSING DESCRIPTION"
46
+
47
+ @classmethod
48
+ @lru_cache(maxsize=1)
49
+ def source_types(cls) -> tuple[type, ...]:
50
+ base_exporter = cls.__orig_bases__[0] # type: ignore[attr-defined]
51
+ source_type = get_args(base_exporter)[0]
52
+ if get_origin(source_type) in [Union, UnionType]:
53
+ return get_args(source_type)
54
+ return (source_type,)
55
+
30
56
 
31
- class CDFExporter(BaseExporter[T_VerifiedRules, T_Export]):
57
+ class CDFExporter(BaseExporter[T_VerifiedRules, T_Export], ABC):
32
58
  @abstractmethod
33
59
  def export_to_cdf_iterable(
34
60
  self, rules: T_VerifiedRules, client: NeatClient, dry_run: bool = False
@@ -4,6 +4,7 @@ from dataclasses import dataclass, field
4
4
  from pathlib import Path
5
5
  from typing import Generic, Literal
6
6
 
7
+ from cognite.client import data_modeling as dm
7
8
  from cognite.client.data_classes._base import (
8
9
  T_CogniteResourceList,
9
10
  T_WritableCogniteResource,
@@ -19,7 +20,7 @@ from cognite.client.exceptions import CogniteAPIError
19
20
 
20
21
  from cognite.neat._client import DataModelingLoader, NeatClient
21
22
  from cognite.neat._client._api.data_modeling_loaders import MultiCogniteAPIError, T_WritableCogniteResourceList
22
- from cognite.neat._client.data_classes.data_modeling import Component
23
+ from cognite.neat._client.data_classes.data_modeling import Component, ViewApplyDict
23
24
  from cognite.neat._client.data_classes.schema import DMSSchema
24
25
  from cognite.neat._issues import IssueList
25
26
  from cognite.neat._issues.warnings import (
@@ -109,6 +110,10 @@ class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
109
110
  self._schema: DMSSchema | None = None
110
111
  self.remove_cdf_spaces = remove_cdf_spaces
111
112
 
113
+ @property
114
+ def description(self) -> str:
115
+ return "Export verified DMS Model to CDF."
116
+
112
117
  def export_to_file(self, rules: DMSRules, filepath: Path) -> None:
113
118
  """Export the rules to a file(s).
114
119
 
@@ -195,6 +200,10 @@ class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
195
200
  ) -> Iterable[UploadResult]:
196
201
  schema = self.export(rules)
197
202
 
203
+ # The CDF UI does not deal well with a child view overwriting a parent property with the same name
204
+ # This is a workaround to remove the duplicated properties
205
+ self._remove_duplicated_properties(schema.views, client)
206
+
198
207
  categorized_items_by_loader = self._categorize_by_loader(client, schema)
199
208
 
200
209
  is_failing = self.existing == "fail" and any(
@@ -372,3 +381,32 @@ class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
372
381
  for data_model in data_models
373
382
  if (data_model.space, data_model.external_id) != (space, external_id)
374
383
  ]
384
+
385
+ @staticmethod
386
+ def _remove_duplicated_properties(views: ViewApplyDict, client: NeatClient) -> None:
387
+ parent_view_ids = {parent for view in views.values() for parent in view.implements}
388
+ parent_view_list = client.data_modeling.views.retrieve(
389
+ list(parent_view_ids), include_inherited_properties=False
390
+ )
391
+ parent_view_by_id = {view.as_id(): view.as_write() for view in parent_view_list}
392
+ for view in views.values():
393
+ if view.implements is None:
394
+ continue
395
+ for parent_id in view.implements:
396
+ if not (parent_view := parent_view_by_id.get(parent_id)):
397
+ continue
398
+ for shared_prop_id in set(view.properties or {}) & set(parent_view.properties or {}):
399
+ if view.properties is None or parent_view.properties is None:
400
+ continue
401
+ prop = view.properties[shared_prop_id]
402
+ parent_prop = parent_view.properties[shared_prop_id]
403
+ if (
404
+ isinstance(prop, dm.MappedPropertyApply)
405
+ and isinstance(parent_prop, dm.MappedPropertyApply)
406
+ and (
407
+ prop.container_property_identifier == parent_prop.container_property_identifier
408
+ and prop.container == parent_prop.container
409
+ and prop.source == parent_prop.source
410
+ )
411
+ ):
412
+ view.properties.pop(shared_prop_id)
@@ -68,6 +68,7 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
68
68
  styling: Style = "default",
69
69
  new_model_id: tuple[str, str] | None = None,
70
70
  sheet_prefix: str | None = None,
71
+ reference_rules_with_prefix: tuple[VerifiedRules, str] | None = None,
71
72
  ):
72
73
  self.sheet_prefix = sheet_prefix or ""
73
74
  if styling not in self.style_options:
@@ -75,6 +76,11 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
75
76
  self.styling = styling
76
77
  self._styling_level = self.style_options.index(styling)
77
78
  self.new_model_id = new_model_id
79
+ self.reference_rules_with_prefix = reference_rules_with_prefix
80
+
81
+ @property
82
+ def description(self) -> str:
83
+ return "Export verified model to Excel."
78
84
 
79
85
  def export_to_file(self, rules: VerifiedRules, filepath: Path) -> None:
80
86
  """Exports transformation rules to excel file."""
@@ -94,6 +100,11 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
94
100
 
95
101
  self._write_metadata_sheet(workbook, dumped_user_rules["Metadata"], sheet_prefix=self.sheet_prefix)
96
102
  self._write_sheets(workbook, dumped_user_rules, rules, sheet_prefix=self.sheet_prefix)
103
+ if self.reference_rules_with_prefix:
104
+ reference_rules, prefix = self.reference_rules_with_prefix
105
+ dumped_reference_rules = reference_rules.dump(entities_exclude_defaults=False, by_alias=True)
106
+ self._write_sheets(workbook, dumped_reference_rules, reference_rules, sheet_prefix=prefix)
107
+ self._write_metadata_sheet(workbook, dumped_reference_rules["Metadata"], sheet_prefix=prefix)
97
108
 
98
109
  if isinstance(rules, InformationRules) and rules.prefixes:
99
110
  self._write_prefixes_sheet(workbook, rules.prefixes)
@@ -218,8 +229,8 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
218
229
  if isinstance(selected_column, MergedCell):
219
230
  selected_column = column_cells[1]
220
231
 
221
- current = sheet.column_dimensions[selected_column.column_letter].width or (max_length + 0.5)
222
- sheet.column_dimensions[selected_column.column_letter].width = min(
232
+ current = sheet.column_dimensions[selected_column.column_letter].width or (max_length + 0.5) # type: ignore[union-attr]
233
+ sheet.column_dimensions[selected_column.column_letter].width = min( # type: ignore[union-attr]
223
234
  max(current, max_length + 0.5), MAX_COLUMN_WIDTH
224
235
  )
225
236
  return None
@@ -42,6 +42,10 @@ class InstanceTemplateExporter(BaseExporter[InformationRules, Workbook]):
42
42
  self.auto_identifier_type = auto_identifier_type
43
43
  self.add_drop_down_list = add_drop_down_list
44
44
 
45
+ @property
46
+ def description(self) -> str:
47
+ return "Export verified information instance template to Excel."
48
+
45
49
  def export(
46
50
  self,
47
51
  rules: InformationRules,
@@ -46,6 +46,10 @@ class OWLExporter(GraphExporter):
46
46
  def export(self, rules: InformationRules) -> Graph:
47
47
  return Ontology.from_rules(rules).as_owl()
48
48
 
49
+ @property
50
+ def description(self) -> str:
51
+ return "Export verified information model to OWL."
52
+
49
53
 
50
54
  class SHACLExporter(GraphExporter):
51
55
  """Exports rules to a SHACL graph."""
@@ -53,13 +57,21 @@ class SHACLExporter(GraphExporter):
53
57
  def export(self, rules: InformationRules) -> Graph:
54
58
  return Ontology.from_rules(rules).as_shacl()
55
59
 
60
+ @property
61
+ def description(self) -> str:
62
+ return "Export verified information model to SHACL."
63
+
56
64
 
57
65
  class SemanticDataModelExporter(GraphExporter):
58
- """Exports verified information rules to a semantic data model."""
66
+ """Exports verified information model to a semantic data model."""
59
67
 
60
68
  def export(self, rules: InformationRules) -> Graph:
61
69
  return Ontology.from_rules(rules).as_semantic_data_model()
62
70
 
71
+ @property
72
+ def description(self) -> str:
73
+ return "Export verified information model to a semantic data model."
74
+
63
75
 
64
76
  class OntologyModel(BaseModel):
65
77
  model_config: ClassVar[ConfigDict] = ConfigDict(arbitrary_types_allowed=True, strict=False, extra="allow")
@@ -43,6 +43,10 @@ class YAMLExporter(BaseExporter[VerifiedRules, str]):
43
43
  self.files = files
44
44
  self.output = output
45
45
 
46
+ @property
47
+ def description(self) -> str:
48
+ return "Export verified model to YAML."
49
+
46
50
  def export_to_file(self, rules: VerifiedRules, filepath: Path) -> None:
47
51
  """Exports transformation rules to YAML/JSON file(s)."""
48
52
  if self.files == "single":
@@ -6,6 +6,7 @@ from datetime import datetime
6
6
  from typing import TYPE_CHECKING, Any, Generic, Literal
7
7
 
8
8
  from pydantic import ValidationError
9
+ from rdflib import URIRef
9
10
 
10
11
  from cognite.neat._constants import DEFAULT_NAMESPACE
11
12
  from cognite.neat._issues import IssueList, NeatError, NeatWarning
@@ -56,6 +57,14 @@ class BaseImporter(ABC, Generic[T_InputRules]):
56
57
 
57
58
  return ProvenanceAgent(id_=DEFAULT_NAMESPACE[f"agent/{type(self).__name__}"])
58
59
 
60
+ @property
61
+ def description(self) -> str:
62
+ return "MISSING DESCRIPTION"
63
+
64
+ @property
65
+ def source_uri(self) -> URIRef:
66
+ return DEFAULT_NAMESPACE["UNKNOWN"]
67
+
59
68
 
60
69
  class _FutureResult:
61
70
  def __init__(self) -> None:
@@ -1,6 +1,6 @@
1
- from collections import Counter, defaultdict
1
+ from collections import defaultdict
2
2
  from collections.abc import Collection, Iterable, Sequence
3
- from datetime import datetime, timezone
3
+ from datetime import datetime
4
4
  from pathlib import Path
5
5
  from typing import Literal, cast
6
6
 
@@ -19,13 +19,20 @@ from cognite.client.data_classes.data_modeling.views import (
19
19
  from cognite.client.utils import ms_to_datetime
20
20
 
21
21
  from cognite.neat._client import NeatClient
22
- from cognite.neat._issues import IssueList, NeatIssue
23
- from cognite.neat._issues.errors import FileTypeUnexpectedError, ResourceMissingIdentifierError, ResourceRetrievalError
22
+ from cognite.neat._issues import IssueList, MultiValueError, NeatIssue
23
+ from cognite.neat._issues.errors import (
24
+ FileTypeUnexpectedError,
25
+ NeatValueError,
26
+ ResourceMissingIdentifierError,
27
+ ResourceRetrievalError,
28
+ )
24
29
  from cognite.neat._issues.warnings import (
30
+ MissingCogniteClientWarning,
25
31
  PropertyNotFoundWarning,
26
32
  PropertyTypeNotSupportedWarning,
27
33
  ResourceNotFoundWarning,
28
34
  ResourcesDuplicatedWarning,
35
+ ResourceUnknownWarning,
29
36
  )
30
37
  from cognite.neat._rules._shared import ReadRules
31
38
  from cognite.neat._rules.importers._base import BaseImporter, _handle_issues
@@ -60,7 +67,6 @@ class DMSImporter(BaseImporter[DMSInputRules]):
60
67
  schema: The schema containing the data model.
61
68
  read_issues: A list of issues that occurred during the import.
62
69
  metadata: Metadata for the data model.
63
- ref_metadata: Metadata for the reference data model.
64
70
 
65
71
  """
66
72
 
@@ -69,25 +75,26 @@ class DMSImporter(BaseImporter[DMSInputRules]):
69
75
  schema: DMSSchema,
70
76
  read_issues: Sequence[NeatIssue] | None = None,
71
77
  metadata: DMSInputMetadata | None = None,
72
- ref_metadata: DMSInputMetadata | None = None,
78
+ referenced_containers: Iterable[dm.ContainerApply] | None = None,
73
79
  ):
74
- # Calling this root schema to distinguish it from
75
- # * User Schema
76
- # * Reference Schema
77
- self.root_schema = schema
80
+ self.schema = schema
78
81
  self.metadata = metadata
79
- self.ref_metadata = ref_metadata
80
82
  self.issue_list = IssueList(read_issues)
81
83
  self._all_containers_by_id = schema.containers.copy()
82
84
  self._all_views_by_id = schema.views.copy()
85
+ if referenced_containers is not None:
86
+ for container in referenced_containers:
87
+ if container.as_id() in self._all_containers_by_id:
88
+ continue
89
+ self._all_containers_by_id[container.as_id()] = container
83
90
 
84
- def update_referenced_containers(self, containers: Iterable[dm.ContainerApply]) -> None:
85
- """Update the referenced containers. This is useful to add Cognite containers identified after the root schema
86
- is read"""
87
- for container in containers:
88
- if container.as_id() in self._all_containers_by_id:
89
- continue
90
- self._all_containers_by_id[container.as_id()] = container
91
+ @property
92
+ def description(self) -> str:
93
+ if self.schema.data_model is not None:
94
+ identifier = f"{self.schema.data_model.as_id().as_tuple()!s}"
95
+ else:
96
+ identifier = "Unknown"
97
+ return f"DMS Data model {identifier} read as unverified data model"
91
98
 
92
99
  @classmethod
93
100
  def from_data_model_id(
@@ -99,8 +106,6 @@ class DMSImporter(BaseImporter[DMSInputRules]):
99
106
 
100
107
  Args:
101
108
  client: Instantiated CogniteClient to retrieve data model.
102
- reference_model_id: The reference data model to retrieve. This is the data model that
103
- the given data model is built on top of, typically, an enterprise data model.
104
109
  data_model_id: Data Model to retrieve.
105
110
 
106
111
  Returns:
@@ -133,7 +138,12 @@ class DMSImporter(BaseImporter[DMSInputRules]):
133
138
 
134
139
  metadata = cls._create_metadata_from_model(user_model)
135
140
 
136
- return cls(schema, issue_list, metadata, None)
141
+ return cls(
142
+ schema,
143
+ issue_list,
144
+ metadata,
145
+ referenced_containers=cls._lookup_referenced_containers(schema, issue_list, client),
146
+ )
137
147
 
138
148
  @classmethod
139
149
  def _find_model_in_list(
@@ -174,15 +184,17 @@ class DMSImporter(BaseImporter[DMSInputRules]):
174
184
  )
175
185
 
176
186
  @classmethod
177
- def from_directory(cls, directory: str | Path) -> "DMSImporter":
187
+ def from_directory(cls, directory: str | Path, client: NeatClient | None = None) -> "DMSImporter":
178
188
  issue_list = IssueList()
179
189
  with _handle_issues(issue_list) as _:
180
190
  schema = DMSSchema.from_directory(directory)
181
191
  # If there were errors during the import, the to_rules
182
- return cls(schema, issue_list)
192
+ return cls(
193
+ schema, issue_list, referenced_containers=cls._lookup_referenced_containers(schema, issue_list, client)
194
+ )
183
195
 
184
196
  @classmethod
185
- def from_zip_file(cls, zip_file: str | Path) -> "DMSImporter":
197
+ def from_zip_file(cls, zip_file: str | Path, client: NeatClient | None = None) -> "DMSImporter":
186
198
  if Path(zip_file).suffix != ".zip":
187
199
  return cls(
188
200
  DMSSchema(),
@@ -191,30 +203,53 @@ class DMSImporter(BaseImporter[DMSInputRules]):
191
203
  issue_list = IssueList()
192
204
  with _handle_issues(issue_list) as _:
193
205
  schema = DMSSchema.from_zip(zip_file)
194
- return cls(schema, issue_list)
206
+ return cls(
207
+ schema, issue_list, referenced_containers=cls._lookup_referenced_containers(schema, issue_list, client)
208
+ )
209
+
210
+ @classmethod
211
+ def _lookup_referenced_containers(
212
+ cls, schema: DMSSchema, issue_list: IssueList, client: NeatClient | None = None
213
+ ) -> Iterable[dm.ContainerApply]:
214
+ ref_containers = schema.externally_referenced_containers()
215
+ if not ref_containers:
216
+ return []
217
+ elif client is None:
218
+ id_ = ""
219
+ if schema.data_model:
220
+ id_ = f" {schema.data_model.as_id()!r}"
221
+ issue_list.append(MissingCogniteClientWarning(f"importing full DMS model{id_}"))
222
+ return []
223
+ return client.loaders.containers.retrieve(list(ref_containers), format="write")
224
+
225
+ @classmethod
226
+ def from_path(cls, path: Path, client: NeatClient | None = None) -> "DMSImporter":
227
+ if path.is_file():
228
+ return cls.from_zip_file(path, client)
229
+ elif path.is_dir():
230
+ return cls.from_directory(path, client)
231
+ else:
232
+ raise NeatValueError(f"Unsupported YAML format: {format}")
195
233
 
196
234
  def to_rules(self) -> ReadRules[DMSInputRules]:
197
235
  if self.issue_list.has_errors:
198
236
  # In case there were errors during the import, the to_rules method will return None
199
- self._end = datetime.now(timezone.utc)
200
- return ReadRules(None, self.issue_list, {})
201
-
202
- if not self.root_schema.data_model:
203
- self.issue_list.append(ResourceMissingIdentifierError("data model", type(self.root_schema).__name__))
204
- self._end = datetime.now(timezone.utc)
205
- return ReadRules(None, self.issue_list, {})
237
+ self.issue_list.trigger_warnings()
238
+ raise MultiValueError(self.issue_list.errors)
206
239
 
207
- model = self.root_schema.data_model
240
+ if not self.schema.data_model:
241
+ self.issue_list.append(ResourceMissingIdentifierError("data model", type(self.schema).__name__))
242
+ self.issue_list.trigger_warnings()
243
+ raise MultiValueError(self.issue_list.errors)
208
244
 
209
- user_rules = self._create_rule_components(
210
- model,
211
- self.root_schema,
212
- self.metadata,
213
- )
245
+ model = self.schema.data_model
214
246
 
215
- self._end = datetime.now(timezone.utc)
247
+ user_rules = self._create_rule_components(model, self.schema, self.metadata)
216
248
 
217
- return ReadRules(user_rules, self.issue_list, {})
249
+ self.issue_list.trigger_warnings()
250
+ if self.issue_list.has_errors:
251
+ raise MultiValueError(self.issue_list.errors)
252
+ return ReadRules(user_rules, {})
218
253
 
219
254
  def _create_rule_components(
220
255
  self,
@@ -222,7 +257,7 @@ class DMSImporter(BaseImporter[DMSInputRules]):
222
257
  schema: DMSSchema,
223
258
  metadata: DMSInputMetadata | None = None,
224
259
  ) -> DMSInputRules:
225
- enum_by_container_property = self._create_enum_collections(schema.containers.values())
260
+ enum_by_container_property = self._create_enum_collections(self._all_containers_by_id.values())
226
261
  enum_collection_by_container_property = {
227
262
  key: enum_list[0].collection for key, enum_list in enum_by_container_property.items() if enum_list
228
263
  }
@@ -255,21 +290,6 @@ class DMSImporter(BaseImporter[DMSInputRules]):
255
290
  enum=[enum for enum_list in enum_by_container_property.values() for enum in enum_list] or None,
256
291
  )
257
292
 
258
- @classmethod
259
- def _create_default_metadata(
260
- cls, views: Sequence[dm.View | dm.ViewApply], is_ref: bool = False
261
- ) -> DMSInputMetadata:
262
- now = datetime.now().replace(microsecond=0)
263
- space = Counter(view.space for view in views).most_common(1)[0][0]
264
- return DMSInputMetadata(
265
- space=space,
266
- external_id="Unknown",
267
- version="0.1.0",
268
- creator="Unknown",
269
- created=now,
270
- updated=now,
271
- )
272
-
273
293
  def _create_dms_property(
274
294
  self,
275
295
  prop_id: str,
@@ -372,8 +392,11 @@ class DMSImporter(BaseImporter[DMSInputRules]):
372
392
  elif isinstance(prop, dm.MappedPropertyApply):
373
393
  container_prop = self._container_prop_unsafe(cast(dm.MappedPropertyApply, prop))
374
394
  if isinstance(container_prop.type, dm.DirectRelation):
375
- if prop.source is None or prop.source not in self._all_views_by_id:
395
+ if prop.source is None:
376
396
  return DMSUnknownEntity()
397
+ elif prop.source not in self._all_views_by_id:
398
+ self.issue_list.append(ResourceUnknownWarning(prop.source, "view", view_entity.as_id(), "view"))
399
+ return ViewEntity.from_id(prop.source)
377
400
  else:
378
401
  return ViewEntity.from_id(prop.source)
379
402
  elif isinstance(container_prop.type, PropertyTypeWithUnit) and container_prop.type.unit:
@@ -5,7 +5,7 @@ from pathlib import Path
5
5
 
6
6
  from pydantic import ValidationError
7
7
 
8
- from cognite.neat._issues import IssueList, NeatIssue
8
+ from cognite.neat._issues import IssueList, MultiValueError, NeatIssue
9
9
  from cognite.neat._issues.warnings import (
10
10
  FileItemNotSupportedWarning,
11
11
  FileMissingRequiredFieldWarning,
@@ -144,5 +144,8 @@ class DTDLImporter(BaseImporter[InformationInputRules]):
144
144
  properties=converter.properties,
145
145
  classes=converter.classes,
146
146
  )
147
+ converter.issues.trigger_warnings()
148
+ if converter.issues.has_errors:
149
+ raise MultiValueError(converter.issues.errors)
147
150
 
148
- return ReadRules(rules, converter.issues, {})
151
+ return ReadRules(rules, {})
@@ -5,7 +5,7 @@ from cognite.client import data_modeling as dm
5
5
  from rdflib import Graph, Namespace, URIRef
6
6
 
7
7
  from cognite.neat._constants import get_default_prefixes_and_namespaces
8
- from cognite.neat._issues import IssueList
8
+ from cognite.neat._issues import IssueList, MultiValueError
9
9
  from cognite.neat._issues.errors import FileReadError
10
10
  from cognite.neat._issues.errors._general import NeatValueError
11
11
  from cognite.neat._rules._shared import ReadRules
@@ -13,9 +13,7 @@ from cognite.neat._rules.importers._base import BaseImporter
13
13
  from cognite.neat._rules.models._base_rules import RoleTypes
14
14
  from cognite.neat._rules.models.data_types import AnyURI
15
15
  from cognite.neat._rules.models.entities import UnknownEntity
16
- from cognite.neat._rules.models.information import (
17
- InformationInputRules,
18
- )
16
+ from cognite.neat._rules.models.information import InformationInputRules
19
17
  from cognite.neat._store import NeatGraphStore
20
18
  from cognite.neat._utils.rdf_ import get_namespace
21
19
 
@@ -50,6 +48,7 @@ class BaseRDFImporter(BaseImporter[InformationInputRules]):
50
48
  max_number_of_instance: int,
51
49
  non_existing_node_type: UnknownEntity | AnyURI,
52
50
  language: str,
51
+ source_name: str = "Unknown",
53
52
  ) -> None:
54
53
  self.issue_list = issue_list
55
54
  self.graph = graph
@@ -60,6 +59,7 @@ class BaseRDFImporter(BaseImporter[InformationInputRules]):
60
59
  self.max_number_of_instance = max_number_of_instance
61
60
  self.non_existing_node_type = non_existing_node_type
62
61
  self.language = language
62
+ self.source_name = source_name
63
63
 
64
64
  @classmethod
65
65
  def from_graph_store(
@@ -87,6 +87,7 @@ class BaseRDFImporter(BaseImporter[InformationInputRules]):
87
87
  max_number_of_instance: int = -1,
88
88
  non_existing_node_type: UnknownEntity | AnyURI = DEFAULT_NON_EXISTING_NODE_TYPE,
89
89
  language: str = "en",
90
+ source_name: str = "Unknown",
90
91
  ):
91
92
  issue_list = IssueList(title=f"{cls.__name__} issues")
92
93
 
@@ -107,6 +108,7 @@ class BaseRDFImporter(BaseImporter[InformationInputRules]):
107
108
  max_number_of_instance=max_number_of_instance,
108
109
  non_existing_node_type=non_existing_node_type,
109
110
  language=language,
111
+ source_name=source_name,
110
112
  )
111
113
 
112
114
  def to_rules(
@@ -115,16 +117,16 @@ class BaseRDFImporter(BaseImporter[InformationInputRules]):
115
117
  """
116
118
  Creates `Rules` object from the data for target role.
117
119
  """
118
-
119
120
  if self.issue_list.has_errors:
120
121
  # In case there were errors during the import, the to_rules method will return None
121
- return ReadRules(None, self.issue_list, {})
122
+ self.issue_list.trigger_warnings()
123
+ raise MultiValueError(self.issue_list.errors)
122
124
 
123
125
  rules_dict = self._to_rules_components()
124
126
 
125
127
  rules = InformationInputRules.load(rules_dict)
126
-
127
- return ReadRules(rules, self.issue_list, {})
128
+ self.issue_list.trigger_warnings()
129
+ return ReadRules(rules, {})
128
130
 
129
131
  def _to_rules_components(self) -> dict:
130
132
  raise NotImplementedError()
@@ -74,6 +74,10 @@ PROPERTIES_QUERY = """
74
74
  class IMFImporter(BaseRDFImporter):
75
75
  """Convert IMF Types provided as SHACL shapes to Input Rules."""
76
76
 
77
+ @property
78
+ def description(self) -> str:
79
+ return f"IMF Types {self.source_name} read as unverified data model"
80
+
77
81
  def _to_rules_components(
78
82
  self,
79
83
  ) -> dict: