cognite-neat 0.97.3__py3-none-any.whl → 0.99.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cognite-neat might be problematic. Click here for more details.

Files changed (109) hide show
  1. cognite/neat/_client/__init__.py +4 -0
  2. cognite/neat/_client/_api/data_modeling_loaders.py +512 -0
  3. cognite/neat/_client/_api/schema.py +50 -0
  4. cognite/neat/_client/_api_client.py +17 -0
  5. cognite/neat/_client/data_classes/__init__.py +0 -0
  6. cognite/neat/{_utils/cdf/data_classes.py → _client/data_classes/data_modeling.py} +8 -135
  7. cognite/neat/{_rules/models/dms/_schema.py → _client/data_classes/schema.py} +32 -281
  8. cognite/neat/_graph/_shared.py +14 -15
  9. cognite/neat/_graph/extractors/_classic_cdf/_assets.py +14 -154
  10. cognite/neat/_graph/extractors/_classic_cdf/_base.py +154 -7
  11. cognite/neat/_graph/extractors/_classic_cdf/_classic.py +23 -12
  12. cognite/neat/_graph/extractors/_classic_cdf/_data_sets.py +17 -92
  13. cognite/neat/_graph/extractors/_classic_cdf/_events.py +13 -162
  14. cognite/neat/_graph/extractors/_classic_cdf/_files.py +15 -179
  15. cognite/neat/_graph/extractors/_classic_cdf/_labels.py +32 -100
  16. cognite/neat/_graph/extractors/_classic_cdf/_relationships.py +27 -178
  17. cognite/neat/_graph/extractors/_classic_cdf/_sequences.py +14 -139
  18. cognite/neat/_graph/extractors/_classic_cdf/_timeseries.py +15 -173
  19. cognite/neat/_graph/extractors/_rdf_file.py +6 -7
  20. cognite/neat/_graph/loaders/__init__.py +1 -2
  21. cognite/neat/_graph/queries/_base.py +17 -1
  22. cognite/neat/_graph/transformers/_classic_cdf.py +50 -134
  23. cognite/neat/_graph/transformers/_prune_graph.py +1 -1
  24. cognite/neat/_graph/transformers/_rdfpath.py +1 -1
  25. cognite/neat/_issues/warnings/__init__.py +6 -0
  26. cognite/neat/_issues/warnings/_external.py +8 -0
  27. cognite/neat/_issues/warnings/_models.py +9 -0
  28. cognite/neat/_issues/warnings/_properties.py +16 -0
  29. cognite/neat/_rules/_constants.py +7 -6
  30. cognite/neat/_rules/_shared.py +3 -8
  31. cognite/neat/_rules/analysis/__init__.py +1 -2
  32. cognite/neat/_rules/analysis/_base.py +10 -27
  33. cognite/neat/_rules/analysis/_dms.py +4 -10
  34. cognite/neat/_rules/analysis/_information.py +2 -10
  35. cognite/neat/_rules/catalog/info-rules-imf.xlsx +0 -0
  36. cognite/neat/_rules/exporters/_base.py +3 -4
  37. cognite/neat/_rules/exporters/_rules2dms.py +29 -40
  38. cognite/neat/_rules/exporters/_rules2excel.py +15 -72
  39. cognite/neat/_rules/exporters/_rules2ontology.py +4 -4
  40. cognite/neat/_rules/importers/_base.py +3 -4
  41. cognite/neat/_rules/importers/_dms2rules.py +21 -45
  42. cognite/neat/_rules/importers/_dtdl2rules/dtdl_converter.py +1 -7
  43. cognite/neat/_rules/importers/_dtdl2rules/dtdl_importer.py +7 -10
  44. cognite/neat/_rules/importers/_rdf/_base.py +17 -29
  45. cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2classes.py +2 -2
  46. cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2metadata.py +5 -10
  47. cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2properties.py +1 -2
  48. cognite/neat/_rules/importers/_rdf/_inference2rules.py +55 -51
  49. cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2classes.py +2 -2
  50. cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2metadata.py +5 -8
  51. cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2properties.py +1 -2
  52. cognite/neat/_rules/importers/_rdf/_shared.py +25 -140
  53. cognite/neat/_rules/importers/_spreadsheet2rules.py +10 -41
  54. cognite/neat/_rules/models/__init__.py +3 -17
  55. cognite/neat/_rules/models/_base_rules.py +118 -62
  56. cognite/neat/_rules/models/dms/__init__.py +2 -2
  57. cognite/neat/_rules/models/dms/_exporter.py +20 -178
  58. cognite/neat/_rules/models/dms/_rules.py +65 -128
  59. cognite/neat/_rules/models/dms/_rules_input.py +72 -56
  60. cognite/neat/_rules/models/dms/_validation.py +16 -109
  61. cognite/neat/_rules/models/entities/_single_value.py +32 -4
  62. cognite/neat/_rules/models/information/_rules.py +19 -122
  63. cognite/neat/_rules/models/information/_rules_input.py +32 -41
  64. cognite/neat/_rules/models/information/_validation.py +34 -102
  65. cognite/neat/_rules/models/mapping/__init__.py +2 -3
  66. cognite/neat/_rules/models/mapping/_classic2core.py +36 -146
  67. cognite/neat/_rules/models/mapping/_classic2core.yaml +339 -0
  68. cognite/neat/_rules/transformers/__init__.py +3 -6
  69. cognite/neat/_rules/transformers/_converters.py +128 -206
  70. cognite/neat/_rules/transformers/_mapping.py +105 -34
  71. cognite/neat/_rules/transformers/_verification.py +5 -16
  72. cognite/neat/_session/_base.py +83 -21
  73. cognite/neat/_session/_collector.py +126 -0
  74. cognite/neat/_session/_drop.py +35 -0
  75. cognite/neat/_session/_inspect.py +22 -10
  76. cognite/neat/_session/_mapping.py +39 -0
  77. cognite/neat/_session/_prepare.py +222 -27
  78. cognite/neat/_session/_read.py +109 -19
  79. cognite/neat/_session/_set.py +2 -2
  80. cognite/neat/_session/_show.py +11 -11
  81. cognite/neat/_session/_to.py +27 -14
  82. cognite/neat/_session/exceptions.py +20 -3
  83. cognite/neat/_store/_base.py +27 -24
  84. cognite/neat/_store/_provenance.py +2 -2
  85. cognite/neat/_utils/auxiliary.py +19 -0
  86. cognite/neat/_utils/rdf_.py +28 -1
  87. cognite/neat/_version.py +1 -1
  88. cognite/neat/_workflows/steps/data_contracts.py +2 -10
  89. cognite/neat/_workflows/steps/lib/current/rules_exporter.py +14 -49
  90. cognite/neat/_workflows/steps/lib/current/rules_importer.py +4 -1
  91. cognite/neat/_workflows/steps/lib/current/rules_validator.py +5 -9
  92. {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/METADATA +4 -3
  93. {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/RECORD +97 -100
  94. cognite/neat/_graph/loaders/_rdf2asset.py +0 -416
  95. cognite/neat/_rules/analysis/_asset.py +0 -173
  96. cognite/neat/_rules/models/asset/__init__.py +0 -13
  97. cognite/neat/_rules/models/asset/_rules.py +0 -109
  98. cognite/neat/_rules/models/asset/_rules_input.py +0 -101
  99. cognite/neat/_rules/models/asset/_validation.py +0 -45
  100. cognite/neat/_rules/models/domain.py +0 -136
  101. cognite/neat/_rules/models/mapping/_base.py +0 -131
  102. cognite/neat/_utils/cdf/loaders/__init__.py +0 -25
  103. cognite/neat/_utils/cdf/loaders/_base.py +0 -54
  104. cognite/neat/_utils/cdf/loaders/_data_modeling.py +0 -339
  105. cognite/neat/_utils/cdf/loaders/_ingestion.py +0 -167
  106. /cognite/neat/{_utils/cdf → _client/_api}/__init__.py +0 -0
  107. {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/LICENSE +0 -0
  108. {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/WHEEL +0 -0
  109. {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/entry_points.txt +0 -0
@@ -0,0 +1,39 @@
1
+ from datetime import datetime, timezone
2
+
3
+ from cognite.neat._rules.models.mapping import load_classic_to_core_mapping
4
+ from cognite.neat._rules.transformers import RuleMapper
5
+ from cognite.neat._store._provenance import Change
6
+
7
+ from ._state import SessionState
8
+ from .exceptions import session_class_wrapper
9
+
10
+
11
+ @session_class_wrapper
12
+ class MappingAPI:
13
+ def __init__(self, state: SessionState):
14
+ self._state = state
15
+
16
+ def classic_to_core(self, org_name: str) -> None:
17
+ """Map classic types to core types.
18
+
19
+ Note this automatically creates an extended CogniteCore model.
20
+
21
+ """
22
+ source_id, rules = self._state.data_model.last_verified_dms_rules
23
+
24
+ start = datetime.now(timezone.utc)
25
+ transformer = RuleMapper(load_classic_to_core_mapping(org_name, rules.metadata.space, rules.metadata.version))
26
+ output = transformer.transform(rules)
27
+ end = datetime.now(timezone.utc)
28
+
29
+ change = Change.from_rules_activity(
30
+ output,
31
+ transformer.agent,
32
+ start,
33
+ end,
34
+ "Mapping classic to core",
35
+ self._state.data_model.provenance.source_entity(source_id)
36
+ or self._state.data_model.provenance.target_entity(source_id),
37
+ )
38
+
39
+ self._state.data_model.write(output.rules, change)
@@ -1,30 +1,48 @@
1
+ import copy
1
2
  from collections.abc import Collection
2
3
  from datetime import datetime, timezone
3
- from typing import Literal
4
+ from typing import Literal, cast
4
5
 
5
6
  from cognite.client.data_classes.data_modeling import DataModelIdentifier
6
7
  from rdflib import URIRef
7
8
 
9
+ from cognite.neat._client import NeatClient
10
+ from cognite.neat._constants import DEFAULT_NAMESPACE
11
+ from cognite.neat._graph.transformers import RelationshipToSchemaTransformer
8
12
  from cognite.neat._graph.transformers._rdfpath import MakeConnectionOnExactMatch
9
- from cognite.neat._rules._shared import ReadRules
13
+ from cognite.neat._rules._shared import InputRules, ReadRules
14
+ from cognite.neat._rules.importers import DMSImporter
15
+ from cognite.neat._rules.models import DMSRules
10
16
  from cognite.neat._rules.models.information._rules_input import InformationInputRules
11
- from cognite.neat._rules.transformers import ReduceCogniteModel, ToCompliantEntities, ToExtension
17
+ from cognite.neat._rules.transformers import (
18
+ PrefixEntities,
19
+ ReduceCogniteModel,
20
+ ToCompliantEntities,
21
+ ToExtension,
22
+ VerifyDMSRules,
23
+ )
24
+ from cognite.neat._store._provenance import Agent as ProvenanceAgent
12
25
  from cognite.neat._store._provenance import Change
13
26
 
14
27
  from ._state import SessionState
15
- from .exceptions import NeatSessionError, intercept_session_exceptions
28
+ from .exceptions import NeatSessionError, session_class_wrapper
16
29
 
30
+ try:
31
+ from rich import print
32
+ except ImportError:
33
+ ...
17
34
 
18
- @intercept_session_exceptions
35
+
36
+ @session_class_wrapper
19
37
  class PrepareAPI:
20
- def __init__(self, state: SessionState, verbose: bool) -> None:
38
+ def __init__(self, client: NeatClient | None, state: SessionState, verbose: bool) -> None:
21
39
  self._state = state
22
40
  self._verbose = verbose
23
- self.data_model = DataModelPrepareAPI(state, verbose)
41
+ self.data_model = DataModelPrepareAPI(client, state, verbose)
24
42
  self.instances = InstancePrepareAPI(state, verbose)
25
43
 
26
44
 
27
- @intercept_session_exceptions
45
+ @session_class_wrapper
28
46
  class InstancePrepareAPI:
29
47
  def __init__(self, state: SessionState, verbose: bool) -> None:
30
48
  self._state = state
@@ -94,34 +112,77 @@ class InstancePrepareAPI:
94
112
  raise NeatSessionError(f"Property {property_} is not defined for type {type_}. Cannot make connection")
95
113
  return type_uri[0], property_uri[0]
96
114
 
115
+ def relationships_as_connections(self, limit: int = 1) -> None:
116
+ """This assumes that you have read a classic CDF knowledge graph including relationships.
97
117
 
98
- @intercept_session_exceptions
118
+ This transformer analyzes the relationships in the graph and modifies them to be part of the schema
119
+ for Assets, Events, Files, Sequences, and TimeSeries. Relationships without any properties
120
+ are replaced by a simple relationship between the source and target nodes. Relationships with
121
+ properties are replaced by a schema that contains the properties as attributes.
122
+
123
+ Args:
124
+ limit: The minimum number of relationships that need to be present for it
125
+ to be converted into a schema. Default is 1.
126
+
127
+ """
128
+ transformer = RelationshipToSchemaTransformer(limit=limit)
129
+ self._state.instances.store.transform(transformer)
130
+
131
+
132
+ @session_class_wrapper
99
133
  class DataModelPrepareAPI:
100
- def __init__(self, state: SessionState, verbose: bool) -> None:
134
+ def __init__(self, client: NeatClient | None, state: SessionState, verbose: bool) -> None:
135
+ self._client = client
101
136
  self._state = state
102
137
  self._verbose = verbose
103
138
 
104
139
  def cdf_compliant_external_ids(self) -> None:
105
140
  """Convert data model component external ids to CDF compliant entities."""
106
- if input := self._state.data_model.last_info_unverified_rule:
107
- source_id, rules = input
141
+ source_id, rules = self._state.data_model.last_info_unverified_rule
142
+
143
+ start = datetime.now(timezone.utc)
144
+ transformer = ToCompliantEntities()
145
+ output: ReadRules[InformationInputRules] = transformer.transform(rules)
146
+ end = datetime.now(timezone.utc)
147
+
148
+ change = Change.from_rules_activity(
149
+ output,
150
+ transformer.agent,
151
+ start,
152
+ end,
153
+ "Converted external ids to CDF compliant entities",
154
+ self._state.data_model.provenance.source_entity(source_id)
155
+ or self._state.data_model.provenance.target_entity(source_id),
156
+ )
108
157
 
109
- start = datetime.now(timezone.utc)
110
- transformer = ToCompliantEntities()
111
- output: ReadRules[InformationInputRules] = transformer.transform(rules)
112
- end = datetime.now(timezone.utc)
158
+ self._state.data_model.write(output, change)
113
159
 
114
- change = Change.from_rules_activity(
115
- output,
116
- transformer.agent,
117
- start,
118
- end,
119
- "Converted external ids to CDF compliant entities",
120
- self._state.data_model.provenance.source_entity(source_id)
121
- or self._state.data_model.provenance.target_entity(source_id),
122
- )
160
+ def prefix(self, prefix: str) -> None:
161
+ """Prefix all views in the data model with the given prefix.
123
162
 
124
- self._state.data_model.write(output, change)
163
+ Args:
164
+ prefix: The prefix to add to the views in the data model.
165
+
166
+ """
167
+ source_id, rules = self._state.data_model.last_unverified_rule
168
+
169
+ start = datetime.now(timezone.utc)
170
+ transformer = PrefixEntities(prefix)
171
+ new_rules = cast(InputRules, copy.deepcopy(rules.get_rules()))
172
+ output = transformer.transform(new_rules)
173
+ end = datetime.now(timezone.utc)
174
+
175
+ change = Change.from_rules_activity(
176
+ output,
177
+ transformer.agent,
178
+ start,
179
+ end,
180
+ "Added prefix to the data model views",
181
+ self._state.data_model.provenance.source_entity(source_id)
182
+ or self._state.data_model.provenance.target_entity(source_id),
183
+ )
184
+
185
+ self._state.data_model.write(output, change)
125
186
 
126
187
  def to_enterprise(
127
188
  self,
@@ -185,7 +246,7 @@ class DataModelPrepareAPI:
185
246
  data_model_id: DataModelIdentifier,
186
247
  org_name: str = "My",
187
248
  mode: Literal["read", "write"] = "read",
188
- dummy_property: str = "dummy",
249
+ dummy_property: str = "GUID",
189
250
  ) -> None:
190
251
  """Uses the current data model as a basis to create solution data model
191
252
 
@@ -235,6 +296,81 @@ class DataModelPrepareAPI:
235
296
 
236
297
  self._state.data_model.write(output.rules, change)
237
298
 
299
+ def to_data_product(
300
+ self,
301
+ data_model_id: DataModelIdentifier,
302
+ org_name: str = "",
303
+ include: Literal["same-space", "all"] = "same-space",
304
+ ) -> None:
305
+ """Uses the current data model as a basis to create data product data model.
306
+
307
+ A data product model is a data model that ONLY maps to containers and do not use implements. This is
308
+ typically used for defining the data in a data product.
309
+
310
+ Args:
311
+ data_model_id: The data product data model id that is being created.
312
+ org_name: Organization name to use for the views in the new data model.
313
+ include: The views to include in the data product data model. Can be either "same-space" or "all".
314
+ If you set same-space, only the views in the same space as the data model will be included.
315
+ """
316
+ source_id, rules = self._state.data_model.last_verified_dms_rules
317
+
318
+ dms_ref: DMSRules | None = None
319
+ view_ids, container_ids = rules.imported_views_and_containers_ids(include_model_views_with_no_properties=True)
320
+ if view_ids or container_ids:
321
+ if self._client is None:
322
+ raise NeatSessionError(
323
+ "No client provided. You are referencing unknown views and containers in your data model, "
324
+ "NEAT needs a client to lookup the definitions. "
325
+ "Please set the client in the session, NeatSession(client=client)."
326
+ )
327
+ schema = self._client.schema.retrieve(list(view_ids), list(container_ids))
328
+
329
+ importer = DMSImporter(schema)
330
+ reference_rules = importer.to_rules().rules
331
+ if reference_rules is not None:
332
+ imported = VerifyDMSRules("continue").transform(reference_rules)
333
+ if dms_ref := imported.rules:
334
+ rules = rules.model_copy(deep=True)
335
+ if rules.containers is None:
336
+ rules.containers = dms_ref.containers
337
+ else:
338
+ existing_containers = {c.container for c in rules.containers}
339
+ rules.containers.extend(
340
+ [c for c in dms_ref.containers or [] if c.container not in existing_containers]
341
+ )
342
+ existing_views = {v.view for v in rules.views}
343
+ rules.views.extend([v for v in dms_ref.views if v.view not in existing_views])
344
+ existing_properties = {(p.view, p.view_property) for p in rules.properties}
345
+ rules.properties.extend(
346
+ [p for p in dms_ref.properties if (p.view, p.view_property) not in existing_properties]
347
+ )
348
+
349
+ start = datetime.now(timezone.utc)
350
+ transformer = ToExtension(
351
+ new_model_id=data_model_id,
352
+ org_name=org_name,
353
+ type_="data_product",
354
+ include=include,
355
+ )
356
+ output = transformer.transform(rules)
357
+ end = datetime.now(timezone.utc)
358
+
359
+ change = Change.from_rules_activity(
360
+ output,
361
+ transformer.agent,
362
+ start,
363
+ end,
364
+ (
365
+ f"Prepared data model {data_model_id} to be data product model "
366
+ f"on top of {rules.metadata.as_data_model_id()}"
367
+ ),
368
+ self._state.data_model.provenance.source_entity(source_id)
369
+ or self._state.data_model.provenance.target_entity(source_id),
370
+ )
371
+
372
+ self._state.data_model.write(output.rules, change)
373
+
238
374
  def reduce(self, drop: Collection[Literal["3D", "Annotation", "BaseViews"] | str]) -> None:
239
375
  """This is a special method that allow you to drop parts of the data model.
240
376
  This only applies to Cognite Data Models.
@@ -267,3 +403,62 @@ class DataModelPrepareAPI:
267
403
  )
268
404
 
269
405
  self._state.data_model.write(output.rules, change)
406
+
407
+ def include_referenced(self) -> None:
408
+ """Include referenced views and containers in the data model."""
409
+ start = datetime.now(timezone.utc)
410
+
411
+ source_id, rules = self._state.data_model.last_verified_dms_rules
412
+ view_ids, container_ids = rules.imported_views_and_containers_ids(include_model_views_with_no_properties=True)
413
+ if not (view_ids or container_ids):
414
+ print(
415
+ f"Data model {rules.metadata.as_data_model_id()} does not have any referenced views or containers."
416
+ f"that is not already included in the data model."
417
+ )
418
+ return
419
+ if self._client is None:
420
+ raise NeatSessionError(
421
+ "No client provided. You are referencing unknown views and containers in your data model, "
422
+ "NEAT needs a client to lookup the definitions. "
423
+ "Please set the client in the session, NeatSession(client=client)."
424
+ )
425
+ schema = self._client.schema.retrieve(list(view_ids), list(container_ids))
426
+ copy_ = rules.model_copy(deep=True)
427
+ copy_.metadata.version = f"{rules.metadata.version}_completed"
428
+ importer = DMSImporter(schema)
429
+ imported = importer.to_rules()
430
+ if imported.rules is None:
431
+ self._state.data_model.issue_lists.append(imported.issues)
432
+ raise NeatSessionError(
433
+ "Could not import the referenced views and containers. "
434
+ "See `neat.inspect.issues()` for more information."
435
+ )
436
+ verified = VerifyDMSRules("continue", post_validate=False).transform(imported.rules)
437
+ if verified.rules is None:
438
+ self._state.data_model.issue_lists.append(verified.issues)
439
+ raise NeatSessionError(
440
+ "Could not verify the referenced views and containers. "
441
+ "See `neat.inspect.issues()` for more information."
442
+ )
443
+ if copy_.containers is None:
444
+ copy_.containers = verified.rules.containers
445
+ else:
446
+ existing_containers = {c.container for c in copy_.containers}
447
+ copy_.containers.extend(
448
+ [c for c in verified.rules.containers or [] if c.container not in existing_containers]
449
+ )
450
+ existing_views = {v.view for v in copy_.views}
451
+ copy_.views.extend([v for v in verified.rules.views if v.view not in existing_views])
452
+ end = datetime.now(timezone.utc)
453
+
454
+ change = Change.from_rules_activity(
455
+ copy_,
456
+ ProvenanceAgent(id_=DEFAULT_NAMESPACE["agent/"]),
457
+ start,
458
+ end,
459
+ (f"Included referenced views and containers in the data model {rules.metadata.as_data_model_id()}"),
460
+ self._state.data_model.provenance.source_entity(source_id)
461
+ or self._state.data_model.provenance.target_entity(source_id),
462
+ )
463
+
464
+ self._state.data_model.write(copy_, change)
@@ -1,17 +1,19 @@
1
1
  import tempfile
2
2
  from datetime import datetime, timezone
3
3
  from pathlib import Path
4
- from typing import Any
4
+ from typing import Any, Literal
5
5
 
6
- from cognite.client import CogniteClient
7
6
  from cognite.client.data_classes.data_modeling import DataModelId, DataModelIdentifier
8
7
 
8
+ from cognite.neat._client import NeatClient
9
+ from cognite.neat._constants import COGNITE_SPACES
9
10
  from cognite.neat._graph import examples as instances_examples
10
11
  from cognite.neat._graph import extractors
11
12
  from cognite.neat._issues import IssueList
12
13
  from cognite.neat._issues.errors import NeatValueError
13
14
  from cognite.neat._rules import importers
14
15
  from cognite.neat._rules._shared import ReadRules
16
+ from cognite.neat._rules.importers import BaseImporter
15
17
  from cognite.neat._store._provenance import Activity as ProvenanceActivity
16
18
  from cognite.neat._store._provenance import Change
17
19
  from cognite.neat._store._provenance import Entity as ProvenanceEntity
@@ -20,23 +22,24 @@ from cognite.neat._utils.reader import GitHubReader, NeatReader, PathReader
20
22
  from ._state import SessionState
21
23
  from ._wizard import NeatObjectType, RDFFileType, object_wizard, rdf_dm_wizard
22
24
  from .engine import import_engine
23
- from .exceptions import NeatSessionError, intercept_session_exceptions
25
+ from .exceptions import NeatSessionError, session_class_wrapper
24
26
 
25
27
 
26
- @intercept_session_exceptions
28
+ @session_class_wrapper
27
29
  class ReadAPI:
28
- def __init__(self, state: SessionState, client: CogniteClient | None, verbose: bool) -> None:
30
+ def __init__(self, state: SessionState, client: NeatClient | None, verbose: bool) -> None:
29
31
  self._state = state
30
32
  self._verbose = verbose
31
33
  self.cdf = CDFReadAPI(state, client, verbose)
32
34
  self.rdf = RDFReadAPI(state, client, verbose)
33
35
  self.excel = ExcelReadAPI(state, client, verbose)
34
36
  self.csv = CSVReadAPI(state, client, verbose)
37
+ self.yaml = YamlReadAPI(state, client, verbose)
35
38
 
36
39
 
37
- @intercept_session_exceptions
40
+ @session_class_wrapper
38
41
  class BaseReadAPI:
39
- def __init__(self, state: SessionState, client: CogniteClient | None, verbose: bool) -> None:
42
+ def __init__(self, state: SessionState, client: NeatClient | None, verbose: bool) -> None:
40
43
  self._state = state
41
44
  self._verbose = verbose
42
45
  self._client = client
@@ -62,14 +65,14 @@ class BaseReadAPI:
62
65
  raise NeatValueError(f"Expected str or Path, got {type(io)}")
63
66
 
64
67
 
65
- @intercept_session_exceptions
68
+ @session_class_wrapper
66
69
  class CDFReadAPI(BaseReadAPI):
67
- def __init__(self, state: SessionState, client: CogniteClient | None, verbose: bool) -> None:
70
+ def __init__(self, state: SessionState, client: NeatClient | None, verbose: bool) -> None:
68
71
  super().__init__(state, client, verbose)
69
72
  self.classic = CDFClassicAPI(state, client, verbose)
70
73
 
71
74
  @property
72
- def _get_client(self) -> CogniteClient:
75
+ def _get_client(self) -> NeatClient:
73
76
  if self._client is None:
74
77
  raise NeatValueError("No client provided. Please provide a client to read a data model.")
75
78
  return self._client
@@ -107,22 +110,59 @@ class CDFReadAPI(BaseReadAPI):
107
110
  return self._store_rules(rules, change)
108
111
 
109
112
 
110
- @intercept_session_exceptions
113
+ @session_class_wrapper
111
114
  class CDFClassicAPI(BaseReadAPI):
112
115
  @property
113
- def _get_client(self) -> CogniteClient:
116
+ def _get_client(self) -> NeatClient:
114
117
  if self._client is None:
115
118
  raise ValueError("No client provided. Please provide a client to read a data model.")
116
119
  return self._client
117
120
 
118
- def assets(self, root_asset_external_id: str) -> None:
119
- extractor = extractors.AssetsExtractor.from_hierarchy(self._get_client, root_asset_external_id)
121
+ def graph(self, root_asset_external_id: str) -> None:
122
+ """Reads the classic knowledge graph from CDF.
123
+
124
+ The Classic Graph consists of the following core resource type.
125
+
126
+ Classic Node CDF Resources:
127
+ - Assets
128
+ - TimeSeries
129
+ - Sequences
130
+ - Events
131
+ - Files
132
+
133
+ All the classic node CDF resources can have one or more connections to one or more assets. This
134
+ will match a direct relationship in the data modeling of CDF.
135
+
136
+ In addition, you have relationships between the classic node CDF resources. This matches an edge
137
+ in the data modeling of CDF.
138
+
139
+ Finally, you have labels and data sets that to organize the graph. In which data sets have a similar,
140
+ but different, role as a space in data modeling. While labels can be compared to node types in data modeling,
141
+ used to quickly filter and find nodes/edges.
142
+
143
+ This extractor will extract the classic CDF graph into Neat starting from either a data set or a root asset.
144
+
145
+ It works as follows:
146
+
147
+ 1. Extract all core nodes (assets, time series, sequences, events, files) filtered by the given data set or
148
+ root asset.
149
+ 2. Extract all relationships starting from any of the extracted core nodes.
150
+ 3. Extract all core nodes that are targets of the relationships that are not already extracted.
151
+ 4. Extract all labels that are connected to the extracted core nodes/relationships.
152
+ 5. Extract all data sets that are connected to the extracted core nodes/relationships.
153
+
154
+ Args:
155
+ root_asset_external_id: The external id of the root asset
156
+
157
+ """
158
+ extractor = extractors.ClassicGraphExtractor(self._get_client, root_asset_external_id=root_asset_external_id)
159
+
120
160
  self._state.instances.store.write(extractor)
121
161
  if self._verbose:
122
- print(f"Asset hierarchy {root_asset_external_id} read successfully")
162
+ print(f"Classic Graph {root_asset_external_id} read successfully")
123
163
 
124
164
 
125
- @intercept_session_exceptions
165
+ @session_class_wrapper
126
166
  class ExcelReadAPI(BaseReadAPI):
127
167
  def __call__(self, io: Any) -> IssueList:
128
168
  reader = NeatReader.create(io)
@@ -142,11 +182,61 @@ class ExcelReadAPI(BaseReadAPI):
142
182
  description=f"Excel file {reader!s} read as unverified data model",
143
183
  )
144
184
  self._store_rules(input_rules, change)
185
+ self._state.data_model.issue_lists.append(input_rules.issues)
186
+ return input_rules.issues
187
+
188
+
189
+ @session_class_wrapper
190
+ class YamlReadAPI(BaseReadAPI):
191
+ def __call__(self, io: Any, format: Literal["neat", "toolkit"] = "neat") -> IssueList:
192
+ reader = NeatReader.create(io)
193
+ if not isinstance(reader, PathReader):
194
+ raise NeatValueError("Only file paths are supported for YAML files")
195
+ start = datetime.now(timezone.utc)
196
+ importer: BaseImporter
197
+ if format == "neat":
198
+ importer = importers.YAMLImporter.from_file(reader.path)
199
+ elif format == "toolkit":
200
+ if reader.path.is_file():
201
+ dms_importer = importers.DMSImporter.from_zip_file(reader.path)
202
+ elif reader.path.is_dir():
203
+ dms_importer = importers.DMSImporter.from_directory(reader.path)
204
+ else:
205
+ raise NeatValueError(f"Unsupported YAML format: {format}")
206
+ ref_containers = dms_importer.root_schema.referenced_container()
207
+ if system_container_ids := [
208
+ container_id for container_id in ref_containers if container_id.space in COGNITE_SPACES
209
+ ]:
210
+ if self._client is None:
211
+ raise NeatSessionError(
212
+ "No client provided. You are referencing Cognite containers in your data model, "
213
+ "NEAT needs a client to lookup the container definitions. "
214
+ "Please set the client in the session, NeatSession(client=client)."
215
+ )
216
+ system_containers = self._client.loaders.containers.retrieve(system_container_ids)
217
+ dms_importer.update_referenced_containers(system_containers)
218
+
219
+ importer = dms_importer
220
+ else:
221
+ raise NeatValueError(f"Unsupported YAML format: {format}")
222
+ input_rules: ReadRules = importer.to_rules()
223
+
224
+ end = datetime.now(timezone.utc)
225
+
226
+ if input_rules.rules:
227
+ change = Change.from_rules_activity(
228
+ input_rules,
229
+ importer.agent,
230
+ start,
231
+ end,
232
+ description=f"YAML file {reader!s} read as unverified data model",
233
+ )
234
+ self._store_rules(input_rules, change)
145
235
 
146
236
  return input_rules.issues
147
237
 
148
238
 
149
- @intercept_session_exceptions
239
+ @session_class_wrapper
150
240
  class CSVReadAPI(BaseReadAPI):
151
241
  def __call__(self, io: Any, type: str, primary_key: str) -> None:
152
242
  reader = NeatReader.create(io)
@@ -167,9 +257,9 @@ class CSVReadAPI(BaseReadAPI):
167
257
  self._state.instances.store.write(extractor)
168
258
 
169
259
 
170
- @intercept_session_exceptions
260
+ @session_class_wrapper
171
261
  class RDFReadAPI(BaseReadAPI):
172
- def __init__(self, state: SessionState, client: CogniteClient | None, verbose: bool) -> None:
262
+ def __init__(self, state: SessionState, client: NeatClient | None, verbose: bool) -> None:
173
263
  super().__init__(state, client, verbose)
174
264
  self.examples = RDFExamples(state)
175
265
 
@@ -7,10 +7,10 @@ from cognite.neat._rules.transformers import SetIDDMSModel
7
7
  from cognite.neat._store._provenance import Change
8
8
 
9
9
  from ._state import SessionState
10
- from .exceptions import NeatSessionError, intercept_session_exceptions
10
+ from .exceptions import NeatSessionError, session_class_wrapper
11
11
 
12
12
 
13
- @intercept_session_exceptions
13
+ @session_class_wrapper
14
14
  class SetAPI:
15
15
  def __init__(self, state: SessionState, verbose: bool) -> None:
16
16
  self._state = state
@@ -16,10 +16,10 @@ from cognite.neat._session.exceptions import NeatSessionError
16
16
  from cognite.neat._utils.rdf_ import remove_namespace_from_uri
17
17
 
18
18
  from ._state import SessionState
19
- from .exceptions import intercept_session_exceptions
19
+ from .exceptions import session_class_wrapper
20
20
 
21
21
 
22
- @intercept_session_exceptions
22
+ @session_class_wrapper
23
23
  class ShowAPI:
24
24
  def __init__(self, state: SessionState) -> None:
25
25
  self._state = state
@@ -27,7 +27,7 @@ class ShowAPI:
27
27
  self.instances = ShowInstanceAPI(self._state)
28
28
 
29
29
 
30
- @intercept_session_exceptions
30
+ @session_class_wrapper
31
31
  class ShowBaseAPI:
32
32
  def __init__(self, state: SessionState) -> None:
33
33
  self._state = state
@@ -63,7 +63,7 @@ class ShowBaseAPI:
63
63
  return net.show(name)
64
64
 
65
65
 
66
- @intercept_session_exceptions
66
+ @session_class_wrapper
67
67
  class ShowDataModelAPI(ShowBaseAPI):
68
68
  def __init__(self, state: SessionState) -> None:
69
69
  super().__init__(state)
@@ -111,7 +111,7 @@ class ShowDataModelAPI(ShowBaseAPI):
111
111
  di_graph.add_edge(
112
112
  prop_.view.suffix,
113
113
  prop_.value_type.suffix,
114
- label=prop_.name or prop_.property_,
114
+ label=prop_.name or prop_.view_property,
115
115
  )
116
116
 
117
117
  return di_graph
@@ -145,7 +145,7 @@ class ShowDataModelAPI(ShowBaseAPI):
145
145
  return di_graph
146
146
 
147
147
 
148
- @intercept_session_exceptions
148
+ @session_class_wrapper
149
149
  class ShowDataModelImplementsAPI(ShowBaseAPI):
150
150
  def __init__(self, state: SessionState) -> None:
151
151
  super().__init__(state)
@@ -206,27 +206,27 @@ class ShowDataModelImplementsAPI(ShowBaseAPI):
206
206
  # if possible use human readable label coming from the view name
207
207
 
208
208
  # add subClassOff as edges
209
- if class_.parent:
209
+ if class_.implements:
210
210
  if not di_graph.has_node(class_.class_.suffix):
211
211
  di_graph.add_node(
212
212
  class_.class_.suffix,
213
213
  label=class_.name or class_.class_.suffix,
214
214
  )
215
215
 
216
- for parent in class_.parent:
216
+ for parent in class_.implements:
217
217
  if not di_graph.has_node(parent.suffix):
218
218
  di_graph.add_node(parent.suffix, label=parent.suffix)
219
219
  di_graph.add_edge(
220
220
  class_.class_.suffix,
221
221
  parent.suffix,
222
- label="subClassOf",
222
+ label="implements",
223
223
  dashes=True,
224
224
  )
225
225
 
226
226
  return di_graph
227
227
 
228
228
 
229
- @intercept_session_exceptions
229
+ @session_class_wrapper
230
230
  class ShowDataModelProvenanceAPI(ShowBaseAPI):
231
231
  def __init__(self, state: SessionState) -> None:
232
232
  super().__init__(state)
@@ -286,7 +286,7 @@ class ShowDataModelProvenanceAPI(ShowBaseAPI):
286
286
  return remove_namespace_from_uri(thing)
287
287
 
288
288
 
289
- @intercept_session_exceptions
289
+ @session_class_wrapper
290
290
  class ShowInstanceAPI(ShowBaseAPI):
291
291
  def __init__(self, state: SessionState) -> None:
292
292
  super().__init__(state)