cognite-neat 0.88.2__py3-none-any.whl → 0.89.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cognite-neat might be problematic. Click here for more details.

Files changed (129) hide show
  1. cognite/neat/_version.py +1 -1
  2. cognite/neat/constants.py +3 -0
  3. cognite/neat/graph/__init__.py +0 -3
  4. cognite/neat/graph/extractors/_mock_graph_generator.py +2 -1
  5. cognite/neat/graph/loaders/_base.py +3 -3
  6. cognite/neat/graph/loaders/_rdf2asset.py +24 -25
  7. cognite/neat/graph/loaders/_rdf2dms.py +20 -15
  8. cognite/neat/issues/__init__.py +1 -3
  9. cognite/neat/issues/_base.py +261 -71
  10. cognite/neat/issues/errors/__init__.py +73 -0
  11. cognite/neat/issues/errors/_external.py +67 -0
  12. cognite/neat/issues/errors/_general.py +35 -0
  13. cognite/neat/issues/errors/_properties.py +62 -0
  14. cognite/neat/issues/errors/_resources.py +111 -0
  15. cognite/neat/issues/errors/_workflow.py +36 -0
  16. cognite/neat/issues/formatters.py +1 -1
  17. cognite/neat/issues/warnings/__init__.py +66 -0
  18. cognite/neat/issues/warnings/_external.py +40 -0
  19. cognite/neat/issues/warnings/_general.py +29 -0
  20. cognite/neat/issues/warnings/_models.py +92 -0
  21. cognite/neat/issues/warnings/_properties.py +44 -0
  22. cognite/neat/issues/warnings/_resources.py +55 -0
  23. cognite/neat/issues/warnings/user_modeling.py +113 -0
  24. cognite/neat/rules/_shared.py +53 -2
  25. cognite/neat/rules/analysis/_base.py +1 -1
  26. cognite/neat/rules/exporters/_base.py +7 -18
  27. cognite/neat/rules/exporters/_rules2dms.py +17 -20
  28. cognite/neat/rules/exporters/_rules2excel.py +9 -16
  29. cognite/neat/rules/exporters/_rules2ontology.py +77 -64
  30. cognite/neat/rules/exporters/_rules2yaml.py +6 -9
  31. cognite/neat/rules/exporters/_validation.py +11 -96
  32. cognite/neat/rules/importers/_base.py +9 -58
  33. cognite/neat/rules/importers/_dms2rules.py +188 -135
  34. cognite/neat/rules/importers/_dtdl2rules/dtdl_converter.py +48 -35
  35. cognite/neat/rules/importers/_dtdl2rules/dtdl_importer.py +36 -45
  36. cognite/neat/rules/importers/_dtdl2rules/spec.py +7 -0
  37. cognite/neat/rules/importers/_rdf/_imf2rules/_imf2classes.py +8 -4
  38. cognite/neat/rules/importers/_rdf/_imf2rules/_imf2metadata.py +3 -3
  39. cognite/neat/rules/importers/_rdf/_imf2rules/_imf2properties.py +18 -11
  40. cognite/neat/rules/importers/_rdf/_imf2rules/_imf2rules.py +12 -19
  41. cognite/neat/rules/importers/_rdf/_inference2rules.py +14 -37
  42. cognite/neat/rules/importers/_rdf/_owl2rules/_owl2classes.py +1 -0
  43. cognite/neat/rules/importers/_rdf/_owl2rules/_owl2properties.py +1 -0
  44. cognite/neat/rules/importers/_rdf/_owl2rules/_owl2rules.py +9 -20
  45. cognite/neat/rules/importers/_rdf/_shared.py +4 -4
  46. cognite/neat/rules/importers/_spreadsheet2rules.py +46 -97
  47. cognite/neat/rules/importers/_yaml2rules.py +32 -58
  48. cognite/neat/rules/models/__init__.py +21 -5
  49. cognite/neat/rules/models/_base_input.py +162 -0
  50. cognite/neat/rules/models/{_base.py → _base_rules.py} +1 -12
  51. cognite/neat/rules/models/_rdfpath.py +4 -4
  52. cognite/neat/rules/models/{_types/_field.py → _types.py} +5 -10
  53. cognite/neat/rules/models/asset/__init__.py +5 -2
  54. cognite/neat/rules/models/asset/_rules.py +3 -23
  55. cognite/neat/rules/models/asset/_rules_input.py +40 -115
  56. cognite/neat/rules/models/asset/_validation.py +14 -10
  57. cognite/neat/rules/models/data_types.py +150 -44
  58. cognite/neat/rules/models/dms/__init__.py +19 -7
  59. cognite/neat/rules/models/dms/_exporter.py +102 -34
  60. cognite/neat/rules/models/dms/_rules.py +65 -162
  61. cognite/neat/rules/models/dms/_rules_input.py +186 -254
  62. cognite/neat/rules/models/dms/_schema.py +87 -78
  63. cognite/neat/rules/models/dms/_serializer.py +44 -3
  64. cognite/neat/rules/models/dms/_validation.py +106 -68
  65. cognite/neat/rules/models/domain.py +52 -1
  66. cognite/neat/rules/models/entities/__init__.py +63 -0
  67. cognite/neat/rules/models/entities/_constants.py +73 -0
  68. cognite/neat/rules/models/entities/_loaders.py +76 -0
  69. cognite/neat/rules/models/entities/_multi_value.py +67 -0
  70. cognite/neat/rules/models/{entities.py → entities/_single_value.py} +74 -232
  71. cognite/neat/rules/models/entities/_types.py +86 -0
  72. cognite/neat/rules/models/{wrapped_entities.py → entities/_wrapped.py} +1 -1
  73. cognite/neat/rules/models/information/__init__.py +10 -2
  74. cognite/neat/rules/models/information/_rules.py +10 -22
  75. cognite/neat/rules/models/information/_rules_input.py +57 -204
  76. cognite/neat/rules/models/information/_validation.py +48 -25
  77. cognite/neat/rules/transformers/__init__.py +21 -0
  78. cognite/neat/rules/transformers/_base.py +81 -0
  79. cognite/neat/rules/{models/information/_converter.py → transformers/_converters.py} +217 -21
  80. cognite/neat/rules/transformers/_map_onto.py +97 -0
  81. cognite/neat/rules/transformers/_pipelines.py +61 -0
  82. cognite/neat/rules/transformers/_verification.py +136 -0
  83. cognite/neat/{graph/stores → store}/_provenance.py +10 -1
  84. cognite/neat/utils/auxiliary.py +2 -35
  85. cognite/neat/utils/cdf/data_classes.py +20 -0
  86. cognite/neat/utils/regex_patterns.py +6 -0
  87. cognite/neat/utils/text.py +17 -0
  88. cognite/neat/workflows/base.py +4 -4
  89. cognite/neat/workflows/cdf_store.py +3 -3
  90. cognite/neat/workflows/steps/data_contracts.py +1 -1
  91. cognite/neat/workflows/steps/lib/current/graph_extractor.py +3 -3
  92. cognite/neat/workflows/steps/lib/current/graph_loader.py +2 -2
  93. cognite/neat/workflows/steps/lib/current/graph_store.py +1 -1
  94. cognite/neat/workflows/steps/lib/current/rules_exporter.py +116 -47
  95. cognite/neat/workflows/steps/lib/current/rules_importer.py +30 -28
  96. cognite/neat/workflows/steps/lib/current/rules_validator.py +5 -6
  97. cognite/neat/workflows/steps/lib/io/io_steps.py +5 -5
  98. cognite/neat/workflows/steps_registry.py +4 -5
  99. {cognite_neat-0.88.2.dist-info → cognite_neat-0.89.0.dist-info}/METADATA +1 -1
  100. {cognite_neat-0.88.2.dist-info → cognite_neat-0.89.0.dist-info}/RECORD +105 -106
  101. cognite/neat/exceptions.py +0 -145
  102. cognite/neat/graph/exceptions.py +0 -90
  103. cognite/neat/issues/errors/external.py +0 -21
  104. cognite/neat/issues/errors/properties.py +0 -75
  105. cognite/neat/issues/errors/resources.py +0 -123
  106. cognite/neat/issues/errors/schema.py +0 -0
  107. cognite/neat/issues/neat_warnings/__init__.py +0 -2
  108. cognite/neat/issues/neat_warnings/identifier.py +0 -27
  109. cognite/neat/issues/neat_warnings/models.py +0 -22
  110. cognite/neat/issues/neat_warnings/properties.py +0 -77
  111. cognite/neat/issues/neat_warnings/resources.py +0 -125
  112. cognite/neat/rules/issues/__init__.py +0 -22
  113. cognite/neat/rules/issues/base.py +0 -63
  114. cognite/neat/rules/issues/dms.py +0 -549
  115. cognite/neat/rules/issues/fileread.py +0 -197
  116. cognite/neat/rules/issues/ontology.py +0 -298
  117. cognite/neat/rules/issues/spreadsheet.py +0 -563
  118. cognite/neat/rules/issues/spreadsheet_file.py +0 -151
  119. cognite/neat/rules/issues/tables.py +0 -72
  120. cognite/neat/rules/models/_constants.py +0 -1
  121. cognite/neat/rules/models/_types/__init__.py +0 -19
  122. cognite/neat/rules/models/asset/_converter.py +0 -4
  123. cognite/neat/rules/models/dms/_converter.py +0 -145
  124. cognite/neat/workflows/_exceptions.py +0 -41
  125. /cognite/neat/{graph/stores → store}/__init__.py +0 -0
  126. /cognite/neat/{graph/stores → store}/_base.py +0 -0
  127. {cognite_neat-0.88.2.dist-info → cognite_neat-0.89.0.dist-info}/LICENSE +0 -0
  128. {cognite_neat-0.88.2.dist-info → cognite_neat-0.89.0.dist-info}/WHEEL +0 -0
  129. {cognite_neat-0.88.2.dist-info → cognite_neat-0.89.0.dist-info}/entry_points.txt +0 -0
@@ -268,6 +268,10 @@ class ViewApplyDict(CogniteResourceDict[ViewId, ViewApply]):
268
268
  def _as_id(cls, resource: ViewApply) -> ViewId:
269
269
  return resource.as_id()
270
270
 
271
+ @classmethod
272
+ def from_iterable(cls, iterable: Iterable[ViewApply]) -> "ViewApplyDict":
273
+ return cls({view.as_id(): view for view in iterable})
274
+
271
275
 
272
276
  class SpaceApplyDict(CogniteResourceDict[str, SpaceApply]):
273
277
  _RESOURCE = SpaceApply
@@ -276,6 +280,10 @@ class SpaceApplyDict(CogniteResourceDict[str, SpaceApply]):
276
280
  def _as_id(cls, resource: SpaceApply) -> str:
277
281
  return resource.space
278
282
 
283
+ @classmethod
284
+ def from_iterable(cls, iterable: Iterable[SpaceApply]) -> "SpaceApplyDict":
285
+ return cls({space.space: space for space in iterable})
286
+
279
287
 
280
288
  class ContainerApplyDict(CogniteResourceDict[ContainerId, ContainerApply]):
281
289
  _RESOURCE = ContainerApply
@@ -284,6 +292,10 @@ class ContainerApplyDict(CogniteResourceDict[ContainerId, ContainerApply]):
284
292
  def _as_id(cls, resource: ContainerApply) -> ContainerId:
285
293
  return resource.as_id()
286
294
 
295
+ @classmethod
296
+ def from_iterable(cls, iterable: Iterable[ContainerApply]) -> "ContainerApplyDict":
297
+ return cls({container.as_id(): container for container in iterable})
298
+
287
299
 
288
300
  class DataModelApplyDict(CogniteResourceDict[DataModelId, DataModelApply]):
289
301
  _RESOURCE = DataModelApply
@@ -292,6 +304,10 @@ class DataModelApplyDict(CogniteResourceDict[DataModelId, DataModelApply]):
292
304
  def _as_id(cls, resource: DataModelApply) -> DataModelId:
293
305
  return resource.as_id()
294
306
 
307
+ @classmethod
308
+ def from_iterable(cls, iterable: Iterable[DataModelApply]) -> "DataModelApplyDict":
309
+ return cls({data_model.as_id(): data_model for data_model in iterable})
310
+
295
311
 
296
312
  class NodeApplyDict(CogniteResourceDict[NodeId, NodeApply]):
297
313
  _RESOURCE = NodeApply
@@ -299,3 +315,7 @@ class NodeApplyDict(CogniteResourceDict[NodeId, NodeApply]):
299
315
  @classmethod
300
316
  def _as_id(cls, resource: NodeApply) -> NodeId:
301
317
  return resource.as_id()
318
+
319
+ @classmethod
320
+ def from_iterable(cls, iterable: Iterable[NodeApply]) -> "NodeApplyDict":
321
+ return cls({node.as_id(): node for node in iterable})
@@ -19,6 +19,12 @@ PROPERTY_ID_COMPLIANCE_REGEX = r"^(\*)|(?!^(Property|property)$)(^[a-zA-Z][a-zA-
19
19
  VERSION_COMPLIANCE_REGEX = r"^[a-zA-Z0-9]([.a-zA-Z0-9_-]{0,41}[a-zA-Z0-9])?$"
20
20
 
21
21
 
22
+ # This pattern ignores commas inside brackets
23
+ SPLIT_ON_COMMA_PATTERN = re.compile(r",(?![^(]*\))")
24
+ # This pattern ignores equal signs inside brackets
25
+ SPLIT_ON_EQUAL_PATTERN = re.compile(r"=(?![^(]*\))")
26
+
27
+
22
28
  class _Patterns:
23
29
  @cached_property
24
30
  def more_than_one_alphanumeric(self) -> re.Pattern:
@@ -1,4 +1,6 @@
1
1
  import re
2
+ from collections.abc import Collection
3
+ from typing import Any
2
4
 
3
5
 
4
6
  def to_camel(string: str) -> str:
@@ -106,3 +108,18 @@ def to_snake(string: str) -> str:
106
108
 
107
109
  def replace_non_alphanumeric_with_underscore(text: str) -> str:
108
110
  return re.sub(r"\W+", "_", text)
111
+
112
+
113
+ def humanize_collection(collection: Collection[Any], /, *, sort: bool = True) -> str:
114
+ if not collection:
115
+ return ""
116
+ elif len(collection) == 1:
117
+ return str(next(iter(collection)))
118
+
119
+ strings = (str(item) for item in collection)
120
+ if sort:
121
+ sequence = sorted(strings)
122
+ else:
123
+ sequence = list(strings)
124
+
125
+ return f"{', '.join(sequence[:-1])} and {sequence[-1]}"
@@ -13,9 +13,9 @@ from prometheus_client import Gauge
13
13
 
14
14
  from cognite.neat.app.monitoring.metrics import NeatMetricsCollector
15
15
  from cognite.neat.config import Config
16
+ from cognite.neat.issues.errors import WorkflowConfigurationNotSetError, WorkflowStepOutputError
16
17
  from cognite.neat.utils.auxiliary import retry_decorator
17
18
  from cognite.neat.workflows import cdf_store, utils
18
- from cognite.neat.workflows._exceptions import ConfigurationNotSet, InvalidStepOutputException
19
19
  from cognite.neat.workflows.cdf_store import CdfStore
20
20
  from cognite.neat.workflows.model import (
21
21
  FlowMessage,
@@ -316,7 +316,7 @@ class BaseWorkflow:
316
316
  elif isinstance(out_obj, DataContract):
317
317
  self.data[type(out_obj).__name__] = out_obj
318
318
  else:
319
- raise InvalidStepOutputException(step_type=type(out_obj).__name__)
319
+ raise WorkflowStepOutputError(step_type=type(out_obj).__name__)
320
320
 
321
321
  elif step.stype == StepType.START_WORKFLOW_TASK_STEP:
322
322
  if self.task_builder:
@@ -510,7 +510,7 @@ class BaseWorkflow:
510
510
  if storage_type == "transformation_rules":
511
511
  self.rules_storage_path = Path(storage_path)
512
512
  if self.default_dataset_id is None:
513
- raise ConfigurationNotSet("default_dataset_id")
513
+ raise WorkflowConfigurationNotSetError("default_dataset_id")
514
514
  self.cdf_store = cdf_store.CdfStore(
515
515
  self.cdf_client, data_set_id=self.default_dataset_id, rules_storage_path=self.rules_storage_path
516
516
  )
@@ -576,7 +576,7 @@ class BaseWorkflow:
576
576
 
577
577
  file_list: list[Path] = []
578
578
  if self.data_store_path is None:
579
- raise ConfigurationNotSet("data_store_path")
579
+ raise WorkflowConfigurationNotSetError("data_store_path")
580
580
  workflow_data_path = Path(self.data_store_path) / "workflows" / self.name
581
581
  try:
582
582
  for root, _dirs, files in os.walk(workflow_data_path):
@@ -11,7 +11,7 @@ from cognite.client.data_classes import Event, FileMetadataUpdate
11
11
  from fastapi.encoders import jsonable_encoder
12
12
  from pydantic import BaseModel
13
13
 
14
- from cognite.neat.workflows._exceptions import ConfigurationNotSet
14
+ from cognite.neat.issues.errors import WorkflowConfigurationNotSetError
15
15
  from cognite.neat.workflows.model import WorkflowFullStateReport, WorkflowState, WorkflowStepEvent
16
16
  from cognite.neat.workflows.utils import get_file_hash
17
17
 
@@ -53,7 +53,7 @@ class CdfStore:
53
53
  def package_workflow(self, workflow_name: str) -> str:
54
54
  """Creates a zip archive from a folder"""
55
55
  if self.workflows_storage_path is None:
56
- raise ConfigurationNotSet("workflows_storage_path")
56
+ raise WorkflowConfigurationNotSetError("workflows_storage_path")
57
57
  folder_path = self.workflows_storage_path / workflow_name
58
58
  archive_path = self.workflows_storage_path / f"{workflow_name}.zip"
59
59
  # Make sure the folder exists
@@ -77,7 +77,7 @@ class CdfStore:
77
77
  # Make sure the archive exists
78
78
  workflow_name = workflow_name.replace(".zip", "")
79
79
  if self.workflows_storage_path is None:
80
- raise ConfigurationNotSet("workflows_storage_path")
80
+ raise WorkflowConfigurationNotSetError("workflows_storage_path")
81
81
  package_full_path = Path(self.workflows_storage_path) / f"{workflow_name}.zip"
82
82
  output_folder = Path(self.workflows_storage_path) / workflow_name
83
83
  if not package_full_path.is_file():
@@ -9,13 +9,13 @@ from cognite.client.data_classes import (
9
9
  )
10
10
  from cognite.client.data_classes.data_modeling import EdgeApply, NodeApply
11
11
 
12
- from cognite.neat.graph.stores import NeatGraphStore
13
12
  from cognite.neat.rules.models import (
14
13
  AssetRules,
15
14
  DMSRules,
16
15
  DomainRules,
17
16
  InformationRules,
18
17
  )
18
+ from cognite.neat.store import NeatGraphStore
19
19
  from cognite.neat.workflows.steps.step_model import DataContract
20
20
 
21
21
 
@@ -8,8 +8,8 @@ from rdflib import URIRef
8
8
  from cognite.neat.constants import DEFAULT_NAMESPACE
9
9
  from cognite.neat.graph.extractors import RdfFileExtractor
10
10
  from cognite.neat.graph.extractors._mock_graph_generator import MockGraphGenerator
11
+ from cognite.neat.issues.errors import WorkflowStepNotInitializedError
11
12
  from cognite.neat.rules._shared import DMSRules, InformationRules
12
- from cognite.neat.workflows._exceptions import StepNotInitialized
13
13
  from cognite.neat.workflows.model import FlowMessage, StepExecutionStatus
14
14
  from cognite.neat.workflows.steps.data_contracts import MultiRuleData, NeatGraph
15
15
  from cognite.neat.workflows.steps.step_model import Configurable, Step
@@ -45,7 +45,7 @@ class GraphFromMockData(Step):
45
45
  self, rules: MultiRuleData, graph_store: NeatGraph
46
46
  ) -> FlowMessage:
47
47
  if self.configs is None:
48
- raise StepNotInitialized(type(self).__name__)
48
+ raise WorkflowStepNotInitializedError(type(self).__name__)
49
49
 
50
50
  if not rules.information and not rules.dms:
51
51
  return FlowMessage(
@@ -110,7 +110,7 @@ class GraphFromRdfFile(Step):
110
110
 
111
111
  def run(self, graph_store: NeatGraph) -> FlowMessage: # type: ignore[override, syntax]
112
112
  if self.configs is None or self.data_store_path is None:
113
- raise StepNotInitialized(type(self).__name__)
113
+ raise WorkflowStepNotInitializedError(type(self).__name__)
114
114
 
115
115
  if source_file := self.configs["File path"]:
116
116
  NeatGraph.graph.write(
@@ -2,7 +2,7 @@ import time
2
2
  from pathlib import Path
3
3
  from typing import ClassVar
4
4
 
5
- from cognite.neat.workflows._exceptions import StepNotInitialized
5
+ from cognite.neat.issues.errors import WorkflowStepNotInitializedError
6
6
  from cognite.neat.workflows.model import FlowMessage
7
7
  from cognite.neat.workflows.steps.data_contracts import NeatGraph
8
8
  from cognite.neat.workflows.steps.step_model import Configurable, Step
@@ -34,7 +34,7 @@ class GraphToRdfFile(Step):
34
34
  self, graph: NeatGraph
35
35
  ) -> FlowMessage: # type: ignore[syntax]
36
36
  if self.configs is None or self.data_store_path is None:
37
- raise StepNotInitialized(type(self).__name__)
37
+ raise WorkflowStepNotInitializedError(type(self).__name__)
38
38
 
39
39
  storage_path = self.data_store_path / Path(self.configs["File path"])
40
40
  relative_graph_file_path = str(storage_path).split("/data/")[1]
@@ -1,6 +1,6 @@
1
1
  from typing import ClassVar
2
2
 
3
- from cognite.neat.graph.stores import NeatGraphStore
3
+ from cognite.neat.store import NeatGraphStore
4
4
  from cognite.neat.workflows.model import FlowMessage
5
5
  from cognite.neat.workflows.steps.data_contracts import (
6
6
  MultiRuleData,
@@ -2,10 +2,17 @@ import time
2
2
  from pathlib import Path
3
3
  from typing import ClassVar, Literal, cast
4
4
 
5
+ from cognite.neat.issues.errors import WorkflowStepNotInitializedError
5
6
  from cognite.neat.rules import exporters
6
- from cognite.neat.rules._shared import DMSRules, InformationRules, Rules
7
- from cognite.neat.rules.models import RoleTypes
8
- from cognite.neat.workflows._exceptions import StepNotInitialized
7
+ from cognite.neat.rules._shared import DMSRules, InformationRules, VerifiedRules
8
+ from cognite.neat.rules.models import AssetRules, RoleTypes
9
+ from cognite.neat.rules.transformers import (
10
+ AssetToInformation,
11
+ DMSToInformation,
12
+ InformationToAsset,
13
+ InformationToDMS,
14
+ RulesPipeline,
15
+ )
9
16
  from cognite.neat.workflows.model import FlowMessage, StepExecutionStatus
10
17
  from cognite.neat.workflows.steps.data_contracts import CogniteClient, MultiRuleData
11
18
  from cognite.neat.workflows.steps.step_model import Configurable, Step
@@ -61,7 +68,7 @@ class DeleteDataModelFromCDF(Step):
61
68
 
62
69
  def run(self, rules: MultiRuleData, cdf_client: CogniteClient) -> FlowMessage: # type: ignore[override]
63
70
  if self.configs is None or self.data_store_path is None:
64
- raise StepNotInitialized(type(self).__name__)
71
+ raise WorkflowStepNotInitializedError(type(self).__name__)
65
72
  components_to_delete = {
66
73
  cast(Literal["all", "spaces", "data_models", "views", "containers"], key)
67
74
  for key, value in self.complex_configs["Components"].items()
@@ -75,26 +82,30 @@ class DeleteDataModelFromCDF(Step):
75
82
  error_text="No DMS Schema components selected for removal! Please select minimum one!",
76
83
  step_execution_status=StepExecutionStatus.ABORT_AND_FAIL,
77
84
  )
78
- input_rules = rules.dms or rules.information
85
+ input_rules = rules.dms or rules.information or rules.asset
79
86
  if input_rules is None:
80
87
  return FlowMessage(
81
88
  error_text="Missing DMS or Information rules in the input data! "
82
89
  "Please ensure that a DMS or Information rules is provided!",
83
90
  step_execution_status=StepExecutionStatus.ABORT_AND_FAIL,
84
91
  )
92
+ if isinstance(input_rules, DMSRules):
93
+ dms_rules = input_rules
94
+ elif isinstance(input_rules, InformationRules):
95
+ dms_rules = InformationToDMS().transform(input_rules).rules
96
+ elif isinstance(input_rules, AssetRules):
97
+ dms_rules = RulesPipeline[AssetRules, DMSRules]([AssetToInformation(), InformationToDMS()]).run(input_rules)
98
+ else:
99
+ raise NotImplementedError(f"Unsupported rules type {type(input_rules)}")
85
100
 
86
101
  dms_exporter = exporters.DMSExporter(
87
102
  export_components=frozenset(components_to_delete),
88
- include_space=(
89
- None
90
- if multi_space_components_delete
91
- else {input_rules.metadata.space if isinstance(input_rules, DMSRules) else input_rules.metadata.prefix}
92
- ),
103
+ include_space=(None if multi_space_components_delete else {dms_rules.metadata.space}),
93
104
  )
94
105
 
95
106
  report_lines = ["# Data Model Deletion from CDF\n\n"]
96
107
  errors = []
97
- for result in dms_exporter.delete_from_cdf(rules=input_rules, client=cdf_client, dry_run=dry_run):
108
+ for result in dms_exporter.delete_from_cdf(rules=dms_rules, client=cdf_client, dry_run=dry_run):
98
109
  report_lines.append(str(result))
99
110
  errors.extend(result.error_messages)
100
111
 
@@ -168,7 +179,7 @@ class RulesToDMS(Step):
168
179
 
169
180
  def run(self, rules: MultiRuleData, cdf_client: CogniteClient) -> FlowMessage: # type: ignore[override]
170
181
  if self.configs is None or self.data_store_path is None:
171
- raise StepNotInitialized(type(self).__name__)
182
+ raise WorkflowStepNotInitializedError(type(self).__name__)
172
183
  existing_components_handling = cast(
173
184
  Literal["fail", "update", "skip", "force"], self.configs["Existing component handling"]
174
185
  )
@@ -192,14 +203,18 @@ class RulesToDMS(Step):
192
203
  "Please ensure that a DMS or Information rules is provided!",
193
204
  step_execution_status=StepExecutionStatus.ABORT_AND_FAIL,
194
205
  )
206
+ if isinstance(input_rules, DMSRules):
207
+ dms_rules = input_rules
208
+ elif isinstance(input_rules, InformationRules):
209
+ dms_rules = InformationToDMS().transform(input_rules).rules
210
+ elif isinstance(input_rules, AssetRules):
211
+ dms_rules = RulesPipeline[AssetRules, DMSRules]([AssetToInformation(), InformationToDMS()]).run(input_rules)
212
+ else:
213
+ raise NotImplementedError(f"Unsupported rules type {type(input_rules)}")
195
214
 
196
215
  dms_exporter = exporters.DMSExporter(
197
216
  export_components=frozenset(components_to_create),
198
- include_space=(
199
- None
200
- if multi_space_components_create
201
- else {input_rules.metadata.space if isinstance(input_rules, DMSRules) else input_rules.metadata.prefix}
202
- ),
217
+ include_space=(None if multi_space_components_create else {dms_rules.metadata.space}),
203
218
  existing_handling=existing_components_handling,
204
219
  )
205
220
 
@@ -212,11 +227,11 @@ class RulesToDMS(Step):
212
227
  )
213
228
  schema_zip = f"{file_name}.zip"
214
229
  schema_full_path = output_dir / schema_zip
215
- dms_exporter.export_to_file(input_rules, schema_full_path)
230
+ dms_exporter.export_to_file(dms_rules, schema_full_path)
216
231
 
217
232
  report_lines = ["# DMS Schema Export to CDF\n\n"]
218
233
  errors = []
219
- for result in dms_exporter.export_to_cdf_iterable(rules=input_rules, client=cdf_client, dry_run=dry_run):
234
+ for result in dms_exporter.export_to_cdf_iterable(rules=dms_rules, client=cdf_client, dry_run=dry_run):
220
235
  report_lines.append(str(result))
221
236
  errors.extend(result.error_messages)
222
237
 
@@ -291,12 +306,12 @@ class RulesToExcel(Step):
291
306
 
292
307
  def run(self, rules: MultiRuleData) -> FlowMessage: # type: ignore[override, syntax]
293
308
  if self.configs is None or self.data_store_path is None:
294
- raise StepNotInitialized(type(self).__name__)
309
+ raise WorkflowStepNotInitializedError(type(self).__name__)
295
310
 
296
311
  dump_format = self.configs.get("Dump Format", "user")
297
312
  styling = cast(exporters.ExcelExporter.Style, self.configs.get("Styling", "default"))
298
313
  role = self.configs.get("Output role format")
299
- output_role = None
314
+ output_role: RoleTypes | None = None
300
315
  if role != "input" and role is not None:
301
316
  output_role = RoleTypes[role]
302
317
 
@@ -310,23 +325,52 @@ class RulesToExcel(Step):
310
325
  step_execution_status=StepExecutionStatus.ABORT_AND_FAIL,
311
326
  )
312
327
 
313
- excel_exporter = exporters.ExcelExporter(
314
- styling=styling,
315
- output_role=output_role,
316
- dump_as=dump_format, # type: ignore[arg-type]
317
- new_model_id=new_model_id,
318
- )
328
+ excel_exporter = exporters.ExcelExporter(styling=styling, dump_as=dump_format, new_model_id=new_model_id) # type: ignore[arg-type]
319
329
 
320
- rule_instance: Rules
330
+ # Todo - Move the conversion to a separate workflow step.
331
+ rule_instance: VerifiedRules
321
332
  if rules.domain:
322
333
  rule_instance = rules.domain
323
334
  elif rules.information:
324
335
  rule_instance = rules.information
325
336
  elif rules.dms:
326
337
  rule_instance = rules.dms
338
+ elif rules.asset:
339
+ rule_instance = rules.asset
327
340
  else:
328
341
  output_errors = "No rules provided for export!"
329
342
  return FlowMessage(error_text=output_errors, step_execution_status=StepExecutionStatus.ABORT_AND_FAIL)
343
+
344
+ if rule_instance.metadata.role is output_role or output_role is None:
345
+ ...
346
+ elif output_role is RoleTypes.dms:
347
+ if isinstance(rule_instance, InformationRules):
348
+ rule_instance = InformationToDMS().transform(rule_instance).rules
349
+ elif isinstance(rule_instance, AssetRules):
350
+ rule_instance = RulesPipeline[AssetRules, DMSRules]([AssetToInformation(), InformationToDMS()]).run(
351
+ rule_instance
352
+ )
353
+ else:
354
+ raise NotImplementedError(f"Role {output_role} is not supported for {type(rules).__name__} rules")
355
+ elif output_role is RoleTypes.information:
356
+ if isinstance(rule_instance, DMSRules):
357
+ rule_instance = DMSToInformation().transform(rule_instance).rules
358
+ elif isinstance(rule_instance, AssetRules):
359
+ rule_instance = AssetToInformation().transform(rule_instance).rules
360
+ else:
361
+ raise NotImplementedError(f"Role {output_role} is not supported for {type(rules).__name__} rules")
362
+ elif output_role is RoleTypes.asset:
363
+ if isinstance(rule_instance, InformationRules):
364
+ rule_instance = InformationToAsset().transform(rule_instance).rules
365
+ elif isinstance(rule_instance, DMSRules):
366
+ rule_instance = RulesPipeline[DMSRules, AssetRules]([DMSToInformation(), InformationToAsset()]).run(
367
+ rule_instance
368
+ )
369
+ else:
370
+ raise NotImplementedError(f"Role {output_role} is not supported for {type(rules).__name__} rules")
371
+ else:
372
+ raise NotImplementedError(f"Role {output_role} is not supported for {type(rules).__name__} rules")
373
+
330
374
  if output_role is None:
331
375
  output_role = rule_instance.metadata.role
332
376
  output_dir = self.data_store_path / Path("staging")
@@ -369,7 +413,7 @@ class RulesToOntology(Step):
369
413
 
370
414
  def run(self, rules: MultiRuleData) -> FlowMessage: # type: ignore[override, syntax]
371
415
  if self.configs is None or self.data_store_path is None:
372
- raise StepNotInitialized(type(self).__name__)
416
+ raise WorkflowStepNotInitializedError(type(self).__name__)
373
417
 
374
418
  if not rules.information and not rules.dms:
375
419
  return FlowMessage(
@@ -384,8 +428,16 @@ class RulesToOntology(Step):
384
428
  )
385
429
  storage_path.parent.mkdir(parents=True, exist_ok=True)
386
430
 
431
+ input_rules = rules.information or rules.dms
432
+ if isinstance(input_rules, DMSRules):
433
+ info_rules = DMSToInformation().transform(input_rules).rules
434
+ elif isinstance(input_rules, InformationRules):
435
+ info_rules = input_rules
436
+ else:
437
+ raise NotImplementedError(f"Unsupported rules type {type(input_rules)}")
438
+
387
439
  exporter = exporters.OWLExporter()
388
- exporter.export_to_file(cast(InformationRules | DMSRules, rules.information or rules.dms), storage_path)
440
+ exporter.export_to_file(info_rules, storage_path)
389
441
 
390
442
  relative_file_path = "/".join(storage_path.relative_to(self.data_store_path).parts)
391
443
 
@@ -420,7 +472,7 @@ class RulesToSHACL(Step):
420
472
 
421
473
  def run(self, rules: MultiRuleData) -> FlowMessage: # type: ignore[override, syntax]
422
474
  if self.configs is None or self.data_store_path is None:
423
- raise StepNotInitialized(type(self).__name__)
475
+ raise WorkflowStepNotInitializedError(type(self).__name__)
424
476
 
425
477
  if not rules.information and not rules.dms:
426
478
  return FlowMessage(
@@ -435,8 +487,16 @@ class RulesToSHACL(Step):
435
487
  )
436
488
  storage_path.parent.mkdir(parents=True, exist_ok=True)
437
489
 
490
+ input_rules = rules.information or rules.dms
491
+ if isinstance(input_rules, DMSRules):
492
+ info_rules = DMSToInformation().transform(input_rules).rules
493
+ elif isinstance(input_rules, InformationRules):
494
+ info_rules = input_rules
495
+ else:
496
+ raise NotImplementedError(f"Unsupported rules type {type(input_rules)}")
497
+
438
498
  exporter = exporters.SHACLExporter()
439
- exporter.export_to_file(cast(InformationRules | DMSRules, rules.information or rules.dms), storage_path)
499
+ exporter.export_to_file(info_rules, storage_path)
440
500
 
441
501
  relative_file_path = "/".join(storage_path.relative_to(self.data_store_path).parts)
442
502
 
@@ -471,7 +531,7 @@ class RulesToSemanticDataModel(Step):
471
531
 
472
532
  def run(self, rules: MultiRuleData) -> FlowMessage: # type: ignore[override, syntax]
473
533
  if self.configs is None or self.data_store_path is None:
474
- raise StepNotInitialized(type(self).__name__)
534
+ raise WorkflowStepNotInitializedError(type(self).__name__)
475
535
 
476
536
  if not rules.information and not rules.dms:
477
537
  return FlowMessage(
@@ -485,9 +545,15 @@ class RulesToSemanticDataModel(Step):
485
545
  self.data_store_path / Path(self.configs["File path"]) if self.configs["File path"] else default_path
486
546
  )
487
547
  storage_path.parent.mkdir(parents=True, exist_ok=True)
488
-
548
+ input_rules = rules.information or rules.dms
549
+ if isinstance(input_rules, DMSRules):
550
+ info_rules = DMSToInformation().transform(input_rules).rules
551
+ elif isinstance(input_rules, InformationRules):
552
+ info_rules = input_rules
553
+ else:
554
+ raise NotImplementedError(f"Unsupported rules type {type(input_rules)}")
489
555
  exporter = exporters.SemanticDataModelExporter()
490
- exporter.export_to_file(cast(InformationRules | DMSRules, rules.information or rules.dms), storage_path)
556
+ exporter.export_to_file(info_rules, storage_path)
491
557
 
492
558
  relative_file_path = "/".join(storage_path.relative_to(self.data_store_path).parts)
493
559
 
@@ -525,7 +591,7 @@ class RulesToCDFTransformations(Step):
525
591
 
526
592
  def run(self, rules: MultiRuleData, cdf_client: CogniteClient) -> FlowMessage: # type: ignore[override]
527
593
  if self.configs is None or self.data_store_path is None:
528
- raise StepNotInitialized(type(self).__name__)
594
+ raise WorkflowStepNotInitializedError(type(self).__name__)
529
595
 
530
596
  input_rules = rules.dms or rules.information
531
597
  if input_rules is None:
@@ -534,28 +600,31 @@ class RulesToCDFTransformations(Step):
534
600
  "Please ensure that a DMS or Information rules is provided!",
535
601
  step_execution_status=StepExecutionStatus.ABORT_AND_FAIL,
536
602
  )
537
- default_instance_space = (
538
- input_rules.metadata.space if isinstance(input_rules, DMSRules) else input_rules.metadata.prefix
539
- )
540
- instance_space = self.configs.get("Instance space") or default_instance_space
603
+ if isinstance(input_rules, DMSRules):
604
+ dms_rules = input_rules
605
+ elif isinstance(input_rules, InformationRules):
606
+ dms_rules = InformationToDMS().transform(input_rules).rules
607
+ elif isinstance(input_rules, AssetRules):
608
+ dms_rules = RulesPipeline[AssetRules, DMSRules]([AssetToInformation(), InformationToDMS()]).run(input_rules)
609
+ else:
610
+ raise NotImplementedError(f"Unsupported rules type {type(input_rules)}")
611
+
612
+ instance_space = self.configs.get("Instance space") or dms_rules.metadata.space
541
613
  dry_run = self.configs.get("Dry run", "False") == "True"
542
614
  dms_exporter = exporters.DMSExporter(
543
615
  export_pipeline=True, instance_space=instance_space, export_components=["spaces"]
544
616
  )
545
617
  output_dir = self.config.staging_path
546
618
  output_dir.mkdir(parents=True, exist_ok=True)
547
- file_name = (
548
- input_rules.metadata.external_id
549
- if isinstance(input_rules, DMSRules)
550
- else input_rules.metadata.name.replace(" ", "_").lower()
551
- )
619
+ file_name = dms_rules.metadata.external_id.replace(":", "_")
552
620
  schema_zip = f"{file_name}_pipeline.zip"
553
621
  schema_full_path = output_dir / schema_zip
554
- dms_exporter.export_to_file(input_rules, schema_full_path)
622
+
623
+ dms_exporter.export_to_file(dms_rules, schema_full_path)
555
624
 
556
625
  report_lines = ["# DMS Schema Export to CDF\n\n"]
557
626
  errors = []
558
- for result in dms_exporter.export_to_cdf_iterable(rules=input_rules, client=cdf_client, dry_run=dry_run):
627
+ for result in dms_exporter.export_to_cdf_iterable(rules=dms_rules, client=cdf_client, dry_run=dry_run):
559
628
  report_lines.append(str(result))
560
629
  errors.extend(result.error_messages)
561
630