cognite-neat 0.99.0__py3-none-any.whl → 0.100.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cognite-neat might be problematic. Click here for more details.

Files changed (84) hide show
  1. cognite/neat/_client/_api/data_modeling_loaders.py +390 -116
  2. cognite/neat/_client/_api/schema.py +63 -2
  3. cognite/neat/_client/data_classes/data_modeling.py +4 -0
  4. cognite/neat/_client/data_classes/schema.py +2 -348
  5. cognite/neat/_constants.py +27 -4
  6. cognite/neat/_graph/extractors/_base.py +7 -0
  7. cognite/neat/_graph/extractors/_classic_cdf/_classic.py +28 -18
  8. cognite/neat/_graph/loaders/_rdf2dms.py +52 -13
  9. cognite/neat/_graph/transformers/__init__.py +3 -3
  10. cognite/neat/_graph/transformers/_classic_cdf.py +135 -56
  11. cognite/neat/_issues/_base.py +26 -17
  12. cognite/neat/_issues/errors/__init__.py +4 -2
  13. cognite/neat/_issues/errors/_external.py +7 -0
  14. cognite/neat/_issues/errors/_properties.py +2 -7
  15. cognite/neat/_issues/errors/_resources.py +1 -1
  16. cognite/neat/_issues/warnings/__init__.py +6 -2
  17. cognite/neat/_issues/warnings/_external.py +9 -1
  18. cognite/neat/_issues/warnings/_resources.py +41 -2
  19. cognite/neat/_issues/warnings/user_modeling.py +4 -4
  20. cognite/neat/_rules/_constants.py +2 -6
  21. cognite/neat/_rules/analysis/_base.py +15 -5
  22. cognite/neat/_rules/analysis/_dms.py +20 -0
  23. cognite/neat/_rules/analysis/_information.py +22 -0
  24. cognite/neat/_rules/exporters/_base.py +3 -5
  25. cognite/neat/_rules/exporters/_rules2dms.py +190 -200
  26. cognite/neat/_rules/importers/__init__.py +1 -3
  27. cognite/neat/_rules/importers/_base.py +1 -1
  28. cognite/neat/_rules/importers/_dms2rules.py +3 -25
  29. cognite/neat/_rules/importers/_rdf/__init__.py +5 -0
  30. cognite/neat/_rules/importers/_rdf/_base.py +34 -11
  31. cognite/neat/_rules/importers/_rdf/_imf2rules.py +91 -0
  32. cognite/neat/_rules/importers/_rdf/_inference2rules.py +40 -7
  33. cognite/neat/_rules/importers/_rdf/_owl2rules.py +80 -0
  34. cognite/neat/_rules/importers/_rdf/_shared.py +138 -441
  35. cognite/neat/_rules/models/_base_rules.py +19 -0
  36. cognite/neat/_rules/models/_types.py +5 -0
  37. cognite/neat/_rules/models/dms/__init__.py +2 -0
  38. cognite/neat/_rules/models/dms/_exporter.py +247 -123
  39. cognite/neat/_rules/models/dms/_rules.py +7 -49
  40. cognite/neat/_rules/models/dms/_rules_input.py +8 -3
  41. cognite/neat/_rules/models/dms/_validation.py +421 -123
  42. cognite/neat/_rules/models/entities/_multi_value.py +3 -0
  43. cognite/neat/_rules/models/information/__init__.py +2 -0
  44. cognite/neat/_rules/models/information/_rules.py +17 -61
  45. cognite/neat/_rules/models/information/_rules_input.py +11 -2
  46. cognite/neat/_rules/models/information/_validation.py +107 -11
  47. cognite/neat/_rules/models/mapping/_classic2core.py +1 -1
  48. cognite/neat/_rules/models/mapping/_classic2core.yaml +8 -4
  49. cognite/neat/_rules/transformers/__init__.py +2 -1
  50. cognite/neat/_rules/transformers/_converters.py +163 -61
  51. cognite/neat/_rules/transformers/_mapping.py +132 -2
  52. cognite/neat/_rules/transformers/_pipelines.py +1 -1
  53. cognite/neat/_rules/transformers/_verification.py +29 -4
  54. cognite/neat/_session/_base.py +46 -60
  55. cognite/neat/_session/_mapping.py +105 -5
  56. cognite/neat/_session/_prepare.py +49 -14
  57. cognite/neat/_session/_read.py +50 -4
  58. cognite/neat/_session/_set.py +1 -0
  59. cognite/neat/_session/_to.py +38 -12
  60. cognite/neat/_session/_wizard.py +5 -0
  61. cognite/neat/_session/engine/_interface.py +3 -2
  62. cognite/neat/_session/exceptions.py +4 -0
  63. cognite/neat/_store/_base.py +79 -19
  64. cognite/neat/_utils/collection_.py +22 -0
  65. cognite/neat/_utils/rdf_.py +30 -4
  66. cognite/neat/_version.py +2 -2
  67. cognite/neat/_workflows/steps/lib/current/rules_exporter.py +3 -91
  68. cognite/neat/_workflows/steps/lib/current/rules_importer.py +2 -16
  69. cognite/neat/_workflows/steps/lib/current/rules_validator.py +3 -5
  70. {cognite_neat-0.99.0.dist-info → cognite_neat-0.100.0.dist-info}/METADATA +1 -1
  71. {cognite_neat-0.99.0.dist-info → cognite_neat-0.100.0.dist-info}/RECORD +74 -82
  72. cognite/neat/_rules/importers/_rdf/_imf2rules/__init__.py +0 -3
  73. cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2classes.py +0 -86
  74. cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2metadata.py +0 -29
  75. cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2properties.py +0 -130
  76. cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2rules.py +0 -154
  77. cognite/neat/_rules/importers/_rdf/_owl2rules/__init__.py +0 -3
  78. cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2classes.py +0 -58
  79. cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2metadata.py +0 -65
  80. cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2properties.py +0 -59
  81. cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2rules.py +0 -39
  82. {cognite_neat-0.99.0.dist-info → cognite_neat-0.100.0.dist-info}/LICENSE +0 -0
  83. {cognite_neat-0.99.0.dist-info → cognite_neat-0.100.0.dist-info}/WHEEL +0 -0
  84. {cognite_neat-0.99.0.dist-info → cognite_neat-0.100.0.dist-info}/entry_points.txt +0 -0
@@ -10,11 +10,12 @@ from cognite.neat._issues import IssueList, catch_issues
10
10
  from cognite.neat._issues.errors import RegexViolationError
11
11
  from cognite.neat._rules import importers
12
12
  from cognite.neat._rules._shared import ReadRules, VerifiedRules
13
- from cognite.neat._rules.importers import DMSImporter
14
- from cognite.neat._rules.models import DMSInputRules, DMSRules, SheetList
13
+ from cognite.neat._rules.models import DMSRules
14
+ from cognite.neat._rules.models.dms import DMSValidation
15
+ from cognite.neat._rules.models.information import InformationValidation
15
16
  from cognite.neat._rules.models.information._rules import InformationRules
16
17
  from cognite.neat._rules.models.information._rules_input import InformationInputRules
17
- from cognite.neat._rules.transformers import ConvertToRules, VerifyAnyRules
18
+ from cognite.neat._rules.transformers import ConvertToRules, InformationToDMS, VerifyAnyRules
18
19
  from cognite.neat._rules.transformers._converters import ConversionTransformer
19
20
  from cognite.neat._store._provenance import (
20
21
  INSTANCES_ENTITY,
@@ -53,7 +54,7 @@ class NeatSession:
53
54
  self.show = ShowAPI(self._state)
54
55
  self.set = SetAPI(self._state, verbose)
55
56
  self.inspect = InspectAPI(self._state)
56
- self.mapping = MappingAPI(self._state)
57
+ self.mapping = MappingAPI(self._state, self._client)
57
58
  self.drop = DropAPI(self._state)
58
59
  self.opt = OptAPI()
59
60
  self.opt._display()
@@ -66,64 +67,40 @@ class NeatSession:
66
67
 
67
68
  def verify(self) -> IssueList:
68
69
  source_id, last_unverified_rule = self._state.data_model.last_unverified_rule
69
-
70
- reference_rules: DMSInputRules | None = None
71
- if isinstance(last_unverified_rule.rules, DMSInputRules):
72
- dms_rules = last_unverified_rule.rules
73
- views_ids, containers_ids = dms_rules.imported_views_and_containers_ids()
74
- if views_ids or containers_ids:
75
- if self._client is None:
76
- raise NeatSessionError(
77
- "No client provided. You are referencing unknown views and containers in your data model, "
78
- "NEAT needs a client to lookup the definitions. "
79
- "Please set the client in the session, NeatSession(client=client)."
80
- )
81
- schema = self._client.schema.retrieve(list(views_ids), list(containers_ids))
82
-
83
- importer = DMSImporter(schema)
84
- reference_rules = importer.to_rules().rules
85
-
86
- if reference_rules is not None:
87
- dms_rules.views.extend(reference_rules.views)
88
- if dms_rules.containers:
89
- dms_rules.containers.extend(reference_rules.containers or [])
90
-
91
- transformer = VerifyAnyRules("continue")
70
+ transformer = VerifyAnyRules("continue", validate=False)
92
71
  start = datetime.now(timezone.utc)
93
72
  output = transformer.try_transform(last_unverified_rule)
94
- end = datetime.now(timezone.utc)
95
73
 
96
74
  if output.rules:
97
- change = Change.from_rules_activity(
98
- output.rules,
99
- transformer.agent,
100
- start,
101
- end,
102
- f"Verified data model {source_id} as {output.rules.metadata.identifier}",
103
- self._state.data_model.provenance.source_entity(source_id)
104
- or self._state.data_model.provenance.target_entity(source_id),
105
- )
106
- if reference_rules is not None and isinstance(output.rules, DMSRules):
107
- # Remove the referenced views and containers from the rules
108
- ref_view_ids = set(reference_rules.as_view_entities())
109
- if ref_view_ids:
110
- output.rules.views = SheetList(
111
- [view for view in output.rules.views if view.view not in ref_view_ids]
112
- )
113
- ref_container_ids = reference_rules.as_container_entities()
114
- if output.rules.containers and ref_container_ids:
115
- output.rules.containers = SheetList(
116
- [
117
- container
118
- for container in output.rules.containers
119
- if container.container not in ref_container_ids
120
- ]
121
- )
122
-
123
- self._state.data_model.write(output.rules, change)
124
-
125
- if isinstance(output.rules, InformationRules):
126
- self._state.instances.store.add_rules(output.rules)
75
+ if isinstance(output.rules, DMSRules):
76
+ issues = DMSValidation(output.rules, self._client).validate()
77
+ elif isinstance(output.rules, InformationRules):
78
+ issues = InformationValidation(output.rules).validate()
79
+ else:
80
+ raise NeatSessionError("Unsupported rule type")
81
+ if issues.has_errors:
82
+ # This is up for discussion, but I think we should not return rules that
83
+ # only pass the verification but not the validation.
84
+ output.rules = None
85
+ output.issues.extend(issues)
86
+
87
+ end = datetime.now(timezone.utc)
88
+
89
+ if output.rules:
90
+ change = Change.from_rules_activity(
91
+ output.rules,
92
+ transformer.agent,
93
+ start,
94
+ end,
95
+ f"Verified data model {source_id} as {output.rules.metadata.identifier}",
96
+ self._state.data_model.provenance.source_entity(source_id)
97
+ or self._state.data_model.provenance.target_entity(source_id),
98
+ )
99
+
100
+ self._state.data_model.write(output.rules, change)
101
+
102
+ if isinstance(output.rules, InformationRules):
103
+ self._state.instances.store.add_rules(output.rules)
127
104
 
128
105
  output.issues.action = "verify"
129
106
  self._state.data_model.issue_lists.append(output.issues)
@@ -131,7 +108,16 @@ class NeatSession:
131
108
  print("You can inspect the issues with the .inspect.issues(...) method.")
132
109
  return output.issues
133
110
 
134
- def convert(self, target: Literal["dms", "information"]) -> IssueList:
111
+ def convert(
112
+ self, target: Literal["dms", "information"], mode: Literal["edge_properties"] | None = None
113
+ ) -> IssueList:
114
+ """Converts the last verified data model to the target type.
115
+
116
+ Args:
117
+ target: The target type to convert the data model to.
118
+ mode: If the target is "dms", the mode to use for the conversion. None is used for default conversion.
119
+ "edge_properties" treas classes that implements Edge as edge properties.
120
+ """
135
121
  start = datetime.now(timezone.utc)
136
122
  issues = IssueList()
137
123
  converter: ConversionTransformer | None = None
@@ -139,7 +125,7 @@ class NeatSession:
139
125
  with catch_issues(issues):
140
126
  if target == "dms":
141
127
  source_id, info_rules = self._state.data_model.last_verified_information_rules
142
- converter = ConvertToRules(DMSRules)
128
+ converter = InformationToDMS(mode=mode)
143
129
  converted_rules = converter.transform(info_rules).rules
144
130
  elif target == "information":
145
131
  source_id, dms_rules = self._state.data_model.last_verified_dms_rules
@@ -1,28 +1,50 @@
1
1
  from datetime import datetime, timezone
2
2
 
3
+ from cognite.neat._client import NeatClient
4
+ from cognite.neat._constants import DEFAULT_NAMESPACE
5
+ from cognite.neat._rules.importers import DMSImporter
6
+ from cognite.neat._rules.models.dms import DMSValidation
3
7
  from cognite.neat._rules.models.mapping import load_classic_to_core_mapping
4
- from cognite.neat._rules.transformers import RuleMapper
8
+ from cognite.neat._rules.transformers import AsParentPropertyId, RuleMapper, VerifyDMSRules
9
+ from cognite.neat._store._provenance import Agent as ProvenanceAgent
5
10
  from cognite.neat._store._provenance import Change
6
11
 
7
12
  from ._state import SessionState
8
- from .exceptions import session_class_wrapper
13
+ from .exceptions import NeatSessionError, session_class_wrapper
9
14
 
10
15
 
11
16
  @session_class_wrapper
12
17
  class MappingAPI:
13
- def __init__(self, state: SessionState):
18
+ def __init__(self, state: SessionState, client: NeatClient | None = None):
19
+ self.data_model = DataModelMappingAPI(state, client)
20
+
21
+
22
+ @session_class_wrapper
23
+ class DataModelMappingAPI:
24
+ def __init__(self, state: SessionState, client: NeatClient | None = None):
14
25
  self._state = state
26
+ self._client = client
15
27
 
16
- def classic_to_core(self, org_name: str) -> None:
28
+ def classic_to_core(self, company_prefix: str, use_parent_property_name: bool = True) -> None:
17
29
  """Map classic types to core types.
18
30
 
19
31
  Note this automatically creates an extended CogniteCore model.
20
32
 
33
+ Args:
34
+ company_prefix: Prefix used for all extended CogniteCore types.
35
+ use_parent_property_name: Whether to use the parent property name in the extended CogniteCore model.
36
+ See below for more information.
37
+
38
+ If you extend CogniteAsset, with for example, ClassicAsset. You will map the property `parentId` to `parent`.
39
+ If you set `user_parent_property_name` to True, the `parentId` will be renamed to `parent` after the
40
+ mapping is done. If you set it to False, the property will remain `parentId`.
21
41
  """
22
42
  source_id, rules = self._state.data_model.last_verified_dms_rules
23
43
 
24
44
  start = datetime.now(timezone.utc)
25
- transformer = RuleMapper(load_classic_to_core_mapping(org_name, rules.metadata.space, rules.metadata.version))
45
+ transformer = RuleMapper(
46
+ load_classic_to_core_mapping(company_prefix, rules.metadata.space, rules.metadata.version)
47
+ )
26
48
  output = transformer.transform(rules)
27
49
  end = datetime.now(timezone.utc)
28
50
 
@@ -37,3 +59,81 @@ class MappingAPI:
37
59
  )
38
60
 
39
61
  self._state.data_model.write(output.rules, change)
62
+
63
+ start = datetime.now(timezone.utc)
64
+
65
+ source_id, rules = self._state.data_model.last_verified_dms_rules
66
+ view_ids, container_ids = DMSValidation(rules, self._client).imported_views_and_containers_ids()
67
+ if not (view_ids or container_ids):
68
+ print(
69
+ f"Data model {rules.metadata.as_data_model_id()} does not have any referenced views or containers."
70
+ f"that is not already included in the data model."
71
+ )
72
+ return
73
+ if self._client is None:
74
+ raise NeatSessionError(
75
+ "No client provided. You are referencing unknown views and containers in your data model, "
76
+ "NEAT needs a client to lookup the definitions. "
77
+ "Please set the client in the session, NeatSession(client=client)."
78
+ )
79
+ schema = self._client.schema.retrieve([v.as_id() for v in view_ids], [c.as_id() for c in container_ids])
80
+ copy_ = rules.model_copy(deep=True)
81
+ copy_.metadata.version = f"{rules.metadata.version}_completed"
82
+ importer = DMSImporter(schema)
83
+ imported = importer.to_rules()
84
+ if imported.rules is None:
85
+ self._state.data_model.issue_lists.append(imported.issues)
86
+ raise NeatSessionError(
87
+ "Could not import the referenced views and containers. "
88
+ "See `neat.inspect.issues()` for more information."
89
+ )
90
+ verified = VerifyDMSRules("continue", validate=False).transform(imported.rules)
91
+ if verified.rules is None:
92
+ self._state.data_model.issue_lists.append(verified.issues)
93
+ raise NeatSessionError(
94
+ "Could not verify the referenced views and containers. "
95
+ "See `neat.inspect.issues()` for more information."
96
+ )
97
+ if copy_.containers is None:
98
+ copy_.containers = verified.rules.containers
99
+ else:
100
+ existing_containers = {c.container for c in copy_.containers}
101
+ copy_.containers.extend(
102
+ [c for c in verified.rules.containers or [] if c.container not in existing_containers]
103
+ )
104
+ existing_views = {v.view for v in copy_.views}
105
+ copy_.views.extend([v for v in verified.rules.views if v.view not in existing_views])
106
+ end = datetime.now(timezone.utc)
107
+
108
+ change = Change.from_rules_activity(
109
+ copy_,
110
+ ProvenanceAgent(id_=DEFAULT_NAMESPACE["agent/"]),
111
+ start,
112
+ end,
113
+ (f"Included referenced views and containers in the data model {rules.metadata.as_data_model_id()}"),
114
+ self._state.data_model.provenance.source_entity(source_id)
115
+ or self._state.data_model.provenance.target_entity(source_id),
116
+ )
117
+
118
+ self._state.data_model.write(copy_, change)
119
+
120
+ if not use_parent_property_name:
121
+ return
122
+
123
+ source_id, rules = self._state.data_model.last_verified_dms_rules
124
+ start = datetime.now(timezone.utc)
125
+ transformer = AsParentPropertyId(self._client)
126
+ output = transformer.transform(rules)
127
+ end = datetime.now(timezone.utc)
128
+
129
+ change = Change.from_rules_activity(
130
+ output,
131
+ transformer.agent,
132
+ start,
133
+ end,
134
+ "Renaming property names to parent name",
135
+ self._state.data_model.provenance.source_entity(source_id)
136
+ or self._state.data_model.provenance.target_entity(source_id),
137
+ )
138
+
139
+ self._state.data_model.write(output.rules, change)
@@ -8,11 +8,13 @@ from rdflib import URIRef
8
8
 
9
9
  from cognite.neat._client import NeatClient
10
10
  from cognite.neat._constants import DEFAULT_NAMESPACE
11
- from cognite.neat._graph.transformers import RelationshipToSchemaTransformer
11
+ from cognite.neat._graph.transformers import RelationshipAsEdgeTransformer
12
12
  from cognite.neat._graph.transformers._rdfpath import MakeConnectionOnExactMatch
13
13
  from cognite.neat._rules._shared import InputRules, ReadRules
14
14
  from cognite.neat._rules.importers import DMSImporter
15
15
  from cognite.neat._rules.models import DMSRules
16
+ from cognite.neat._rules.models.dms import DMSValidation
17
+ from cognite.neat._rules.models.entities import ClassEntity
16
18
  from cognite.neat._rules.models.information._rules_input import InformationInputRules
17
19
  from cognite.neat._rules.transformers import (
18
20
  PrefixEntities,
@@ -112,20 +114,23 @@ class InstancePrepareAPI:
112
114
  raise NeatSessionError(f"Property {property_} is not defined for type {type_}. Cannot make connection")
113
115
  return type_uri[0], property_uri[0]
114
116
 
115
- def relationships_as_connections(self, limit: int = 1) -> None:
117
+ def relationships_as_edges(self, min_relationship_types: int = 1, limit_per_type: int | None = None) -> None:
116
118
  """This assumes that you have read a classic CDF knowledge graph including relationships.
117
119
 
118
- This transformer analyzes the relationships in the graph and modifies them to be part of the schema
119
- for Assets, Events, Files, Sequences, and TimeSeries. Relationships without any properties
120
- are replaced by a simple relationship between the source and target nodes. Relationships with
121
- properties are replaced by a schema that contains the properties as attributes.
120
+ This method converts relationships into edges in the graph. This is useful as the
121
+ edges will be picked up as part of the schema connected to Assets, Events, Files, Sequences,
122
+ and TimeSeries in the InferenceImporter.
122
123
 
123
124
  Args:
124
- limit: The minimum number of relationships that need to be present for it
125
- to be converted into a schema. Default is 1.
125
+ min_relationship_types: The minimum number of relationship types that must exists to convert those
126
+ relationships to edges. For example, if there is only 5 relationships between Assets and TimeSeries,
127
+ and limit is 10, those relationships will not be converted to edges.
128
+ limit_per_type: The number of conversions to perform per relationship type. For example, if there are 10
129
+ relationships between Assets and TimeSeries, and limit_per_type is 1, only 1 of those relationships
130
+ will be converted to an edge. If None, all relationships will be converted.
126
131
 
127
132
  """
128
- transformer = RelationshipToSchemaTransformer(limit=limit)
133
+ transformer = RelationshipAsEdgeTransformer(min_relationship_types, limit_per_type)
129
134
  self._state.instances.store.transform(transformer)
130
135
 
131
136
 
@@ -316,7 +321,7 @@ class DataModelPrepareAPI:
316
321
  source_id, rules = self._state.data_model.last_verified_dms_rules
317
322
 
318
323
  dms_ref: DMSRules | None = None
319
- view_ids, container_ids = rules.imported_views_and_containers_ids(include_model_views_with_no_properties=True)
324
+ view_ids, container_ids = DMSValidation(rules, self._client).imported_views_and_containers_ids()
320
325
  if view_ids or container_ids:
321
326
  if self._client is None:
322
327
  raise NeatSessionError(
@@ -324,7 +329,7 @@ class DataModelPrepareAPI:
324
329
  "NEAT needs a client to lookup the definitions. "
325
330
  "Please set the client in the session, NeatSession(client=client)."
326
331
  )
327
- schema = self._client.schema.retrieve(list(view_ids), list(container_ids))
332
+ schema = self._client.schema.retrieve([v.as_id() for v in view_ids], [c.as_id() for c in container_ids])
328
333
 
329
334
  importer = DMSImporter(schema)
330
335
  reference_rules = importer.to_rules().rules
@@ -409,7 +414,7 @@ class DataModelPrepareAPI:
409
414
  start = datetime.now(timezone.utc)
410
415
 
411
416
  source_id, rules = self._state.data_model.last_verified_dms_rules
412
- view_ids, container_ids = rules.imported_views_and_containers_ids(include_model_views_with_no_properties=True)
417
+ view_ids, container_ids = DMSValidation(rules, self._client).imported_views_and_containers_ids()
413
418
  if not (view_ids or container_ids):
414
419
  print(
415
420
  f"Data model {rules.metadata.as_data_model_id()} does not have any referenced views or containers."
@@ -422,7 +427,7 @@ class DataModelPrepareAPI:
422
427
  "NEAT needs a client to lookup the definitions. "
423
428
  "Please set the client in the session, NeatSession(client=client)."
424
429
  )
425
- schema = self._client.schema.retrieve(list(view_ids), list(container_ids))
430
+ schema = self._client.schema.retrieve([v.as_id() for v in view_ids], [c.as_id() for c in container_ids])
426
431
  copy_ = rules.model_copy(deep=True)
427
432
  copy_.metadata.version = f"{rules.metadata.version}_completed"
428
433
  importer = DMSImporter(schema)
@@ -433,7 +438,7 @@ class DataModelPrepareAPI:
433
438
  "Could not import the referenced views and containers. "
434
439
  "See `neat.inspect.issues()` for more information."
435
440
  )
436
- verified = VerifyDMSRules("continue", post_validate=False).transform(imported.rules)
441
+ verified = VerifyDMSRules("continue", validate=False).transform(imported.rules)
437
442
  if verified.rules is None:
438
443
  self._state.data_model.issue_lists.append(verified.issues)
439
444
  raise NeatSessionError(
@@ -462,3 +467,33 @@ class DataModelPrepareAPI:
462
467
  )
463
468
 
464
469
  self._state.data_model.write(copy_, change)
470
+
471
+ def add_implements_to_classes(self, suffix: Literal["Edge"], implements: str = "Edge") -> None:
472
+ """All classes with the suffix will have the implements property set to the given value.
473
+
474
+ Args:
475
+ suffix: The suffix of the classes to add the implements property to.
476
+ implements: The value of the implements property to set.
477
+
478
+ """
479
+ source_id, rules = self._state.data_model.last_verified_information_rules
480
+ start = datetime.now(timezone.utc)
481
+
482
+ output = rules.model_copy(deep=True)
483
+ for class_ in output.classes:
484
+ if class_.class_.suffix.endswith(suffix):
485
+ class_.implements = [ClassEntity(prefix=class_.class_.prefix, suffix=implements)]
486
+ output.metadata.version = f"{rules.metadata.version}.implements_{implements}"
487
+ end = datetime.now(timezone.utc)
488
+
489
+ change = Change.from_rules_activity(
490
+ output,
491
+ ProvenanceAgent(id_=DEFAULT_NAMESPACE["agent/"]),
492
+ start,
493
+ end,
494
+ (f"Added implements property to classes with suffix {suffix}"),
495
+ self._state.data_model.provenance.source_entity(source_id)
496
+ or self._state.data_model.provenance.target_entity(source_id),
497
+ )
498
+
499
+ self._state.data_model.write(output, change)
@@ -20,7 +20,7 @@ from cognite.neat._store._provenance import Entity as ProvenanceEntity
20
20
  from cognite.neat._utils.reader import GitHubReader, NeatReader, PathReader
21
21
 
22
22
  from ._state import SessionState
23
- from ._wizard import NeatObjectType, RDFFileType, object_wizard, rdf_dm_wizard
23
+ from ._wizard import NeatObjectType, RDFFileType, XMLFileType, object_wizard, rdf_dm_wizard, xml_format_wizard
24
24
  from .engine import import_engine
25
25
  from .exceptions import NeatSessionError, session_class_wrapper
26
26
 
@@ -35,6 +35,7 @@ class ReadAPI:
35
35
  self.excel = ExcelReadAPI(state, client, verbose)
36
36
  self.csv = CSVReadAPI(state, client, verbose)
37
37
  self.yaml = YamlReadAPI(state, client, verbose)
38
+ self.xml = XMLReadAPI(state, client, verbose)
38
39
 
39
40
 
40
41
  @session_class_wrapper
@@ -118,7 +119,7 @@ class CDFClassicAPI(BaseReadAPI):
118
119
  raise ValueError("No client provided. Please provide a client to read a data model.")
119
120
  return self._client
120
121
 
121
- def graph(self, root_asset_external_id: str) -> None:
122
+ def graph(self, root_asset_external_id: str, limit_per_type: int | None = None) -> None:
122
123
  """Reads the classic knowledge graph from CDF.
123
124
 
124
125
  The Classic Graph consists of the following core resource type.
@@ -153,9 +154,12 @@ class CDFClassicAPI(BaseReadAPI):
153
154
 
154
155
  Args:
155
156
  root_asset_external_id: The external id of the root asset
157
+ limit_per_type: The maximum number of nodes to extract per core node type. If None, all nodes are extracted.
156
158
 
157
159
  """
158
- extractor = extractors.ClassicGraphExtractor(self._get_client, root_asset_external_id=root_asset_external_id)
160
+ extractor = extractors.ClassicGraphExtractor(
161
+ self._get_client, root_asset_external_id=root_asset_external_id, limit_per_type=limit_per_type
162
+ )
159
163
 
160
164
  self._state.instances.store.write(extractor)
161
165
  if self._verbose:
@@ -248,7 +252,7 @@ class CSVReadAPI(BaseReadAPI):
248
252
  else:
249
253
  raise NeatValueError("Only file paths are supported for CSV files")
250
254
  engine = import_engine()
251
- engine.set.source = ".csv"
255
+ engine.set.format = "csv"
252
256
  engine.set.file = path
253
257
  engine.set.type = type
254
258
  engine.set.primary_key = primary_key
@@ -257,6 +261,48 @@ class CSVReadAPI(BaseReadAPI):
257
261
  self._state.instances.store.write(extractor)
258
262
 
259
263
 
264
+ @session_class_wrapper
265
+ class XMLReadAPI(BaseReadAPI):
266
+ def __call__(
267
+ self,
268
+ io: Any,
269
+ format: XMLFileType | None = None,
270
+ ) -> None:
271
+ reader = NeatReader.create(io)
272
+ if isinstance(reader, GitHubReader):
273
+ path = Path(tempfile.gettempdir()).resolve() / reader.name
274
+ path.write_text(reader.read_text())
275
+ elif isinstance(reader, PathReader):
276
+ path = reader.path
277
+ else:
278
+ raise NeatValueError("Only file paths are supported for XML files")
279
+ if format is None:
280
+ format = xml_format_wizard()
281
+
282
+ if format.lower() == "dexpi":
283
+ return self.dexpi(path)
284
+
285
+ if format.lower() == "aml":
286
+ return self.aml(path)
287
+
288
+ else:
289
+ raise NeatValueError("Only support XML files of DEXPI format at the moment.")
290
+
291
+ def dexpi(self, path):
292
+ engine = import_engine()
293
+ engine.set.format = "dexpi"
294
+ engine.set.file = path
295
+ extractor = engine.create_extractor()
296
+ self._state.instances.store.write(extractor)
297
+
298
+ def aml(self, path):
299
+ engine = import_engine()
300
+ engine.set.format = "aml"
301
+ engine.set.file = path
302
+ extractor = engine.create_extractor()
303
+ self._state.instances.store.write(extractor)
304
+
305
+
260
306
  @session_class_wrapper
261
307
  class RDFReadAPI(BaseReadAPI):
262
308
  def __init__(self, state: SessionState, client: NeatClient | None, verbose: bool) -> None:
@@ -29,6 +29,7 @@ class SetAPI:
29
29
 
30
30
  start = datetime.now(timezone.utc)
31
31
  transformer = SetIDDMSModel(new_model_id)
32
+
32
33
  output = transformer.transform(rules)
33
34
  end = datetime.now(timezone.utc)
34
35
 
@@ -1,3 +1,4 @@
1
+ from collections.abc import Collection
1
2
  from pathlib import Path
2
3
  from typing import Any, Literal, overload
3
4
 
@@ -9,6 +10,7 @@ from cognite.neat._issues import IssueList, catch_warnings
9
10
  from cognite.neat._rules import exporters
10
11
  from cognite.neat._rules._constants import PATTERNS
11
12
  from cognite.neat._rules._shared import VerifiedRules
13
+ from cognite.neat._rules.exporters._rules2dms import Component
12
14
  from cognite.neat._utils.upload import UploadResultCore, UploadResultList
13
15
 
14
16
  from ._state import SessionState
@@ -51,7 +53,24 @@ class ToAPI:
51
53
  @overload
52
54
  def yaml(self, io: Any, format: Literal["neat", "toolkit"] = "neat") -> None: ...
53
55
 
54
- def yaml(self, io: Any | None = None, format: Literal["neat", "toolkit"] = "neat") -> str | None:
56
+ def yaml(
57
+ self, io: Any | None = None, format: Literal["neat", "toolkit"] = "neat", skip_system_spaces: bool = True
58
+ ) -> str | None:
59
+ """Export the verified data model to YAML.
60
+
61
+ Args:
62
+ io: The file path or file-like object to write the YAML file to. Defaults to None.
63
+ format: The format of the YAML file. Defaults to "neat".
64
+ skip_system_spaces: If True, system spaces will be skipped. Defaults to True.
65
+
66
+ ... note::
67
+
68
+ - "neat": This is the format Neat uses to store the data model.
69
+ - "toolkit": This is the format used by Cognite Toolkit, that matches the CDF API.
70
+
71
+ Returns:
72
+ str | None: If io is None, the YAML string will be returned. Otherwise, None will be returned.
73
+ """
55
74
  if format == "neat":
56
75
  exporter = exporters.YAMLExporter()
57
76
  last_verified = self._state.data_model.last_verified_rule[1]
@@ -66,9 +85,12 @@ class ToAPI:
66
85
  "This is required for the 'toolkit' format."
67
86
  )
68
87
  dms_rule = self._state.data_model.last_verified_dms_rules[1]
69
- exporters.DMSExporter().export_to_file(dms_rule, Path(io))
88
+ user_path = Path(io)
89
+ if user_path.suffix == "" and not user_path.exists():
90
+ user_path.mkdir(parents=True)
91
+ exporters.DMSExporter(remove_cdf_spaces=skip_system_spaces).export_to_file(dms_rule, user_path)
70
92
  else:
71
- raise NeatSessionError("Please provide a valid format. {['neat', 'toolkit']}")
93
+ raise NeatSessionError("Please provide a valid format. 'neat' or 'toolkit'")
72
94
 
73
95
  return None
74
96
 
@@ -106,36 +128,40 @@ class CDFToAPI:
106
128
 
107
129
  def data_model(
108
130
  self,
109
- existing_handling: Literal["fail", "skip", "update", "force"] = "skip",
131
+ existing: Literal["fail", "skip", "update", "force", "recreate"] = "update",
110
132
  dry_run: bool = False,
111
- fallback_one_by_one: bool = False,
133
+ drop_data: bool = False,
134
+ components: Component | Collection[Component] | None = None,
112
135
  ):
113
136
  """Export the verified DMS data model to CDF.
114
137
 
115
138
  Args:
116
- existing_handling: How to handle if component of data model exists. Defaults to "skip".
139
+ existing: What to do if the component already exists. Defaults to "update".
140
+ See the note below for more information about the options.
117
141
  dry_run: If True, no changes will be made to CDF. Defaults to False.
118
- fallback_one_by_one: If True, will fall back to one-by-one upload if batch upload fails. Defaults to False.
142
+ drop_data: If existing is 'force' or 'recreate' and the operation will lead to data loss,
143
+ the component will be skipped unless drop_data is True. Defaults to False.
144
+ Note this only applies to spaces and containers if they contain data.
145
+ components: The components to export. If None, all components will be exported. Defaults to None.
119
146
 
120
147
  ... note::
121
148
 
122
149
  - "fail": If any component already exists, the export will fail.
123
150
  - "skip": If any component already exists, it will be skipped.
124
151
  - "update": If any component already exists, it will be updated.
125
- - "force": If any component already exists, it will be deleted and recreated.
152
+ - "force": If any component already exists, and the update fails, it will be deleted and recreated.
153
+ - "recreate": All components will be deleted and recreated.
126
154
 
127
155
  """
128
156
 
129
- exporter = exporters.DMSExporter(existing_handling=existing_handling)
157
+ exporter = exporters.DMSExporter(existing=existing, export_components=components, drop_data=drop_data)
130
158
 
131
159
  if not self._client:
132
160
  raise NeatSessionError("No client provided!")
133
161
 
134
162
  conversion_issues = IssueList(action="to.cdf.data_model")
135
163
  with catch_warnings(conversion_issues):
136
- result = exporter.export_to_cdf(
137
- self._state.data_model.last_verified_dms_rules[1], self._client, dry_run, fallback_one_by_one
138
- )
164
+ result = exporter.export_to_cdf(self._state.data_model.last_verified_dms_rules[1], self._client, dry_run)
139
165
  result.insert(0, UploadResultCore(name="schema", issues=conversion_issues))
140
166
  self._state.data_model.outcome.append(result)
141
167
  print("You can inspect the details with the .inspect.data_model.outcome(...) method.")
@@ -7,12 +7,17 @@ from cognite.neat._rules._constants import PATTERNS
7
7
 
8
8
  RDFFileType = Literal["Ontology", "IMF Types", "Inference"]
9
9
  NeatObjectType = Literal["Data Model", "Instances"]
10
+ XMLFileType = Literal["dexpi", "aml"]
10
11
 
11
12
 
12
13
  def object_wizard(message: str = "Select object") -> NeatObjectType:
13
14
  return _selection(message, get_args(NeatObjectType))
14
15
 
15
16
 
17
+ def xml_format_wizard(message: str = "Select XML format") -> XMLFileType:
18
+ return _selection(message, get_args(XMLFileType))
19
+
20
+
16
21
  def rdf_dm_wizard(message: str = "Select source:") -> RDFFileType:
17
22
  return _selection(message, get_args(RDFFileType))
18
23
 
@@ -9,14 +9,15 @@ class Extractor(Protocol):
9
9
 
10
10
 
11
11
  class ConfigAPI(Protocol):
12
- source: str | None
12
+ format: str | None = None
13
13
  file: Any | None
14
14
  type: str | None
15
15
  primary_key: str | None
16
+ mapping: Any | None = None
16
17
 
17
18
 
18
19
  class NeatEngine(Protocol):
19
- version: str = "1.0.0"
20
+ version: str = "2.0.0"
20
21
 
21
22
  @property
22
23
  def set(self) -> ConfigAPI: ...