cognite-neat 0.97.3__py3-none-any.whl → 0.99.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cognite-neat might be problematic. Click here for more details.

Files changed (109) hide show
  1. cognite/neat/_client/__init__.py +4 -0
  2. cognite/neat/_client/_api/data_modeling_loaders.py +512 -0
  3. cognite/neat/_client/_api/schema.py +50 -0
  4. cognite/neat/_client/_api_client.py +17 -0
  5. cognite/neat/_client/data_classes/__init__.py +0 -0
  6. cognite/neat/{_utils/cdf/data_classes.py → _client/data_classes/data_modeling.py} +8 -135
  7. cognite/neat/{_rules/models/dms/_schema.py → _client/data_classes/schema.py} +32 -281
  8. cognite/neat/_graph/_shared.py +14 -15
  9. cognite/neat/_graph/extractors/_classic_cdf/_assets.py +14 -154
  10. cognite/neat/_graph/extractors/_classic_cdf/_base.py +154 -7
  11. cognite/neat/_graph/extractors/_classic_cdf/_classic.py +23 -12
  12. cognite/neat/_graph/extractors/_classic_cdf/_data_sets.py +17 -92
  13. cognite/neat/_graph/extractors/_classic_cdf/_events.py +13 -162
  14. cognite/neat/_graph/extractors/_classic_cdf/_files.py +15 -179
  15. cognite/neat/_graph/extractors/_classic_cdf/_labels.py +32 -100
  16. cognite/neat/_graph/extractors/_classic_cdf/_relationships.py +27 -178
  17. cognite/neat/_graph/extractors/_classic_cdf/_sequences.py +14 -139
  18. cognite/neat/_graph/extractors/_classic_cdf/_timeseries.py +15 -173
  19. cognite/neat/_graph/extractors/_rdf_file.py +6 -7
  20. cognite/neat/_graph/loaders/__init__.py +1 -2
  21. cognite/neat/_graph/queries/_base.py +17 -1
  22. cognite/neat/_graph/transformers/_classic_cdf.py +50 -134
  23. cognite/neat/_graph/transformers/_prune_graph.py +1 -1
  24. cognite/neat/_graph/transformers/_rdfpath.py +1 -1
  25. cognite/neat/_issues/warnings/__init__.py +6 -0
  26. cognite/neat/_issues/warnings/_external.py +8 -0
  27. cognite/neat/_issues/warnings/_models.py +9 -0
  28. cognite/neat/_issues/warnings/_properties.py +16 -0
  29. cognite/neat/_rules/_constants.py +7 -6
  30. cognite/neat/_rules/_shared.py +3 -8
  31. cognite/neat/_rules/analysis/__init__.py +1 -2
  32. cognite/neat/_rules/analysis/_base.py +10 -27
  33. cognite/neat/_rules/analysis/_dms.py +4 -10
  34. cognite/neat/_rules/analysis/_information.py +2 -10
  35. cognite/neat/_rules/catalog/info-rules-imf.xlsx +0 -0
  36. cognite/neat/_rules/exporters/_base.py +3 -4
  37. cognite/neat/_rules/exporters/_rules2dms.py +29 -40
  38. cognite/neat/_rules/exporters/_rules2excel.py +15 -72
  39. cognite/neat/_rules/exporters/_rules2ontology.py +4 -4
  40. cognite/neat/_rules/importers/_base.py +3 -4
  41. cognite/neat/_rules/importers/_dms2rules.py +21 -45
  42. cognite/neat/_rules/importers/_dtdl2rules/dtdl_converter.py +1 -7
  43. cognite/neat/_rules/importers/_dtdl2rules/dtdl_importer.py +7 -10
  44. cognite/neat/_rules/importers/_rdf/_base.py +17 -29
  45. cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2classes.py +2 -2
  46. cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2metadata.py +5 -10
  47. cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2properties.py +1 -2
  48. cognite/neat/_rules/importers/_rdf/_inference2rules.py +55 -51
  49. cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2classes.py +2 -2
  50. cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2metadata.py +5 -8
  51. cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2properties.py +1 -2
  52. cognite/neat/_rules/importers/_rdf/_shared.py +25 -140
  53. cognite/neat/_rules/importers/_spreadsheet2rules.py +10 -41
  54. cognite/neat/_rules/models/__init__.py +3 -17
  55. cognite/neat/_rules/models/_base_rules.py +118 -62
  56. cognite/neat/_rules/models/dms/__init__.py +2 -2
  57. cognite/neat/_rules/models/dms/_exporter.py +20 -178
  58. cognite/neat/_rules/models/dms/_rules.py +65 -128
  59. cognite/neat/_rules/models/dms/_rules_input.py +72 -56
  60. cognite/neat/_rules/models/dms/_validation.py +16 -109
  61. cognite/neat/_rules/models/entities/_single_value.py +32 -4
  62. cognite/neat/_rules/models/information/_rules.py +19 -122
  63. cognite/neat/_rules/models/information/_rules_input.py +32 -41
  64. cognite/neat/_rules/models/information/_validation.py +34 -102
  65. cognite/neat/_rules/models/mapping/__init__.py +2 -3
  66. cognite/neat/_rules/models/mapping/_classic2core.py +36 -146
  67. cognite/neat/_rules/models/mapping/_classic2core.yaml +339 -0
  68. cognite/neat/_rules/transformers/__init__.py +3 -6
  69. cognite/neat/_rules/transformers/_converters.py +128 -206
  70. cognite/neat/_rules/transformers/_mapping.py +105 -34
  71. cognite/neat/_rules/transformers/_verification.py +5 -16
  72. cognite/neat/_session/_base.py +83 -21
  73. cognite/neat/_session/_collector.py +126 -0
  74. cognite/neat/_session/_drop.py +35 -0
  75. cognite/neat/_session/_inspect.py +22 -10
  76. cognite/neat/_session/_mapping.py +39 -0
  77. cognite/neat/_session/_prepare.py +222 -27
  78. cognite/neat/_session/_read.py +109 -19
  79. cognite/neat/_session/_set.py +2 -2
  80. cognite/neat/_session/_show.py +11 -11
  81. cognite/neat/_session/_to.py +27 -14
  82. cognite/neat/_session/exceptions.py +20 -3
  83. cognite/neat/_store/_base.py +27 -24
  84. cognite/neat/_store/_provenance.py +2 -2
  85. cognite/neat/_utils/auxiliary.py +19 -0
  86. cognite/neat/_utils/rdf_.py +28 -1
  87. cognite/neat/_version.py +1 -1
  88. cognite/neat/_workflows/steps/data_contracts.py +2 -10
  89. cognite/neat/_workflows/steps/lib/current/rules_exporter.py +14 -49
  90. cognite/neat/_workflows/steps/lib/current/rules_importer.py +4 -1
  91. cognite/neat/_workflows/steps/lib/current/rules_validator.py +5 -9
  92. {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/METADATA +4 -3
  93. {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/RECORD +97 -100
  94. cognite/neat/_graph/loaders/_rdf2asset.py +0 -416
  95. cognite/neat/_rules/analysis/_asset.py +0 -173
  96. cognite/neat/_rules/models/asset/__init__.py +0 -13
  97. cognite/neat/_rules/models/asset/_rules.py +0 -109
  98. cognite/neat/_rules/models/asset/_rules_input.py +0 -101
  99. cognite/neat/_rules/models/asset/_validation.py +0 -45
  100. cognite/neat/_rules/models/domain.py +0 -136
  101. cognite/neat/_rules/models/mapping/_base.py +0 -131
  102. cognite/neat/_utils/cdf/loaders/__init__.py +0 -25
  103. cognite/neat/_utils/cdf/loaders/_base.py +0 -54
  104. cognite/neat/_utils/cdf/loaders/_data_modeling.py +0 -339
  105. cognite/neat/_utils/cdf/loaders/_ingestion.py +0 -167
  106. /cognite/neat/{_utils/cdf → _client/_api}/__init__.py +0 -0
  107. {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/LICENSE +0 -0
  108. {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/WHEEL +0 -0
  109. {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/entry_points.txt +0 -0
@@ -1,9 +1,9 @@
1
1
  from pathlib import Path
2
2
  from typing import Any, Literal, overload
3
3
 
4
- from cognite.client import CogniteClient
5
4
  from cognite.client.data_classes.data_modeling import SpaceApply
6
5
 
6
+ from cognite.neat._client import NeatClient
7
7
  from cognite.neat._graph import loaders
8
8
  from cognite.neat._issues import IssueList, catch_warnings
9
9
  from cognite.neat._rules import exporters
@@ -12,12 +12,12 @@ from cognite.neat._rules._shared import VerifiedRules
12
12
  from cognite.neat._utils.upload import UploadResultCore, UploadResultList
13
13
 
14
14
  from ._state import SessionState
15
- from .exceptions import NeatSessionError, intercept_session_exceptions
15
+ from .exceptions import NeatSessionError, session_class_wrapper
16
16
 
17
17
 
18
- @intercept_session_exceptions
18
+ @session_class_wrapper
19
19
  class ToAPI:
20
- def __init__(self, state: SessionState, client: CogniteClient | None, verbose: bool) -> None:
20
+ def __init__(self, state: SessionState, client: NeatClient | None, verbose: bool) -> None:
21
21
  self._state = state
22
22
  self._verbose = verbose
23
23
  self.cdf = CDFToAPI(state, client, verbose)
@@ -46,23 +46,36 @@ class ToAPI:
46
46
  return None
47
47
 
48
48
  @overload
49
- def yaml(self, io: None) -> str: ...
49
+ def yaml(self, io: None, format: Literal["neat"] = "neat") -> str: ...
50
50
 
51
51
  @overload
52
- def yaml(self, io: Any) -> None: ...
53
-
54
- def yaml(self, io: Any | None = None) -> str | None:
55
- exporter = exporters.YAMLExporter()
56
- if io is None:
57
- return exporter.export(self._state.data_model.last_verified_rule[1])
52
+ def yaml(self, io: Any, format: Literal["neat", "toolkit"] = "neat") -> None: ...
53
+
54
+ def yaml(self, io: Any | None = None, format: Literal["neat", "toolkit"] = "neat") -> str | None:
55
+ if format == "neat":
56
+ exporter = exporters.YAMLExporter()
57
+ last_verified = self._state.data_model.last_verified_rule[1]
58
+ if io is None:
59
+ return exporter.export(last_verified)
60
+
61
+ exporter.export_to_file(last_verified, Path(io))
62
+ elif format == "toolkit":
63
+ if io is None or not isinstance(io, str | Path):
64
+ raise NeatSessionError(
65
+ "Please provide a zip file or directory path to write the YAML files to."
66
+ "This is required for the 'toolkit' format."
67
+ )
68
+ dms_rule = self._state.data_model.last_verified_dms_rules[1]
69
+ exporters.DMSExporter().export_to_file(dms_rule, Path(io))
70
+ else:
71
+ raise NeatSessionError("Please provide a valid format. {['neat', 'toolkit']}")
58
72
 
59
- exporter.export_to_file(self._state.data_model.last_verified_rule[1], Path(io))
60
73
  return None
61
74
 
62
75
 
63
- @intercept_session_exceptions
76
+ @session_class_wrapper
64
77
  class CDFToAPI:
65
- def __init__(self, state: SessionState, client: CogniteClient | None, verbose: bool) -> None:
78
+ def __init__(self, state: SessionState, client: NeatClient | None, verbose: bool) -> None:
66
79
  self._client = client
67
80
  self._state = state
68
81
  self._verbose = verbose
@@ -2,6 +2,8 @@ import functools
2
2
  from collections.abc import Callable
3
3
  from typing import Any
4
4
 
5
+ from ._collector import _COLLECTOR
6
+
5
7
  try:
6
8
  from rich import print
7
9
 
@@ -16,9 +18,10 @@ class NeatSessionError(Exception):
16
18
  ...
17
19
 
18
20
 
19
- def _intercept_session_exceptions(func: Callable):
21
+ def _session_method_wrapper(func: Callable, cls_name: str):
20
22
  @functools.wraps(func)
21
23
  def wrapper(*args: Any, **kwargs: Any):
24
+ _COLLECTOR.track_session_command(f"{cls_name}.{func.__name__}", *args, **kwargs)
22
25
  try:
23
26
  return func(*args, **kwargs)
24
27
  except NeatSessionError as e:
@@ -40,7 +43,21 @@ def _intercept_session_exceptions(func: Callable):
40
43
  return wrapper
41
44
 
42
45
 
43
- def intercept_session_exceptions(cls: type):
46
+ def session_class_wrapper(cls: type):
47
+ """This decorator wraps all methods of a class.
48
+
49
+ It should be used with all composition classes used with the NeatSession class.
50
+
51
+ It does the following:
52
+ * Intercepts all NeatSession exceptions and prints them in a user-friendly way.
53
+ * Collects user metrics.
54
+
55
+ Args:
56
+ cls: NeatSession composition class
57
+
58
+ Returns:
59
+ cls: NeatSession composition class with all methods wrapped
60
+ """
44
61
  to_check = [cls]
45
62
  while to_check:
46
63
  cls = to_check.pop()
@@ -49,7 +66,7 @@ def intercept_session_exceptions(cls: type):
49
66
  continue
50
67
  attr = getattr(cls, attr_name)
51
68
  if callable(attr):
52
- setattr(cls, attr_name, _intercept_session_exceptions(attr))
69
+ setattr(cls, attr_name, _session_method_wrapper(attr, cls.__name__))
53
70
  elif isinstance(attr, type):
54
71
  to_check.append(attr)
55
72
  return cls
@@ -7,10 +7,11 @@ from typing import cast
7
7
 
8
8
  import pandas as pd
9
9
  from pandas import Index
10
- from rdflib import Graph, Namespace, URIRef
10
+ from rdflib import Dataset, Namespace, URIRef
11
11
  from rdflib.plugins.stores.sparqlstore import SPARQLUpdateStore
12
12
 
13
13
  from cognite.neat._constants import DEFAULT_NAMESPACE
14
+ from cognite.neat._graph._shared import rdflib_to_oxi_type
14
15
  from cognite.neat._graph.extractors import RdfFileExtractor, TripleExtractors
15
16
  from cognite.neat._graph.queries import Queries
16
17
  from cognite.neat._graph.transformers import Transformers
@@ -42,7 +43,7 @@ class NeatGraphStore:
42
43
 
43
44
  def __init__(
44
45
  self,
45
- graph: Graph,
46
+ graph: Dataset,
46
47
  rules: InformationRules | None = None,
47
48
  ):
48
49
  self.rules: InformationRules | None = None
@@ -109,7 +110,7 @@ class NeatGraphStore:
109
110
 
110
111
  @classmethod
111
112
  def from_memory_store(cls, rules: InformationRules | None = None) -> "Self":
112
- return cls(Graph(identifier=DEFAULT_NAMESPACE), rules)
113
+ return cls(Dataset(), rules)
113
114
 
114
115
  @classmethod
115
116
  def from_sparql_store(
@@ -127,7 +128,7 @@ class NeatGraphStore:
127
128
  postAsEncoded=False,
128
129
  autocommit=False,
129
130
  )
130
- graph = Graph(store=store, identifier=DEFAULT_NAMESPACE)
131
+ graph = Dataset(store=store)
131
132
  return cls(graph, rules)
132
133
 
133
134
  @classmethod
@@ -150,9 +151,8 @@ class NeatGraphStore:
150
151
  else:
151
152
  raise Exception("Error initializing Oxigraph store")
152
153
 
153
- graph = Graph(
154
+ graph = Dataset(
154
155
  store=oxrdflib.OxigraphStore(store=oxi_store),
155
- identifier=DEFAULT_NAMESPACE,
156
156
  )
157
157
 
158
158
  return cls(graph, rules)
@@ -162,7 +162,7 @@ class NeatGraphStore:
162
162
  success = True
163
163
 
164
164
  if isinstance(extractor, RdfFileExtractor) and not extractor.issue_list.has_errors:
165
- self._parse_file(extractor.filepath, cast(str, extractor.mime_type), extractor.base_uri)
165
+ self._parse_file(extractor.filepath, cast(str, extractor.format), extractor.base_uri)
166
166
  elif isinstance(extractor, RdfFileExtractor):
167
167
  success = False
168
168
  issue_text = "\n".join([issue.as_message() for issue in extractor.issue_list])
@@ -244,33 +244,36 @@ class NeatGraphStore:
244
244
  def _parse_file(
245
245
  self,
246
246
  filepath: Path,
247
- mime_type: str = "application/rdf+xml",
247
+ format: str = "turtle",
248
248
  base_uri: URIRef | None = None,
249
249
  ) -> None:
250
250
  """Imports graph data from file.
251
251
 
252
252
  Args:
253
253
  filepath : File path to file containing graph data, by default None
254
- mime_type : MIME type of graph data, by default "application/rdf+xml"
255
- base_uri : Add base IRI to graph, by default True
254
+ format : rdflib format file containing RDF graph, by default "turtle"
255
+ base_uri : base URI to add to graph in case of relative URIs, by default None
256
+
257
+ !!! note "Oxigraph store"
258
+ By default we are using non-transactional mode for parsing RDF files.
259
+ This gives us a significant performance boost when importing large RDF files.
260
+ Underhood of rdflib we are triggering oxrdflib plugin which in respect
261
+ calls `bulk_load` method from oxigraph store. See more at:
262
+ https://pyoxigraph.readthedocs.io/en/stable/store.html#pyoxigraph.Store.bulk_load
256
263
  """
257
264
 
258
265
  # Oxigraph store, do not want to type hint this as it is an optional dependency
259
266
  if type(self.graph.store).__name__ == "OxigraphStore":
260
-
261
- def parse_to_oxi_store():
262
- local_import("pyoxigraph", "oxi")
263
- import pyoxigraph
264
-
265
- cast(pyoxigraph.Store, self.graph.store._store).bulk_load(
266
- str(filepath),
267
- mime_type,
268
- base_iri=base_uri,
269
- to_graph=pyoxigraph.NamedNode(self.graph.identifier),
270
- )
271
- cast(pyoxigraph.Store, self.graph.store._store).optimize()
272
-
273
- parse_to_oxi_store()
267
+ local_import("pyoxigraph", "oxi")
268
+
269
+ # this is necessary to trigger rdflib oxigraph plugin
270
+ self.graph.parse(
271
+ filepath,
272
+ format=rdflib_to_oxi_type(format),
273
+ transactional=False,
274
+ publicID=base_uri,
275
+ )
276
+ self.graph.store._store.optimize() # type: ignore[attr-defined]
274
277
 
275
278
  # All other stores
276
279
  else:
@@ -93,14 +93,14 @@ class Entity:
93
93
  return cls(
94
94
  was_attributed_to=agent,
95
95
  was_generated_by=activity,
96
- id_=rules.id_,
96
+ id_=rules.metadata.identifier,
97
97
  )
98
98
 
99
99
  elif isinstance(rules, ReadRules | JustRules) and rules.rules is not None:
100
100
  return cls(
101
101
  was_attributed_to=agent,
102
102
  was_generated_by=activity,
103
- id_=rules.rules.id_,
103
+ id_=rules.rules.metadata.identifier,
104
104
  )
105
105
  else:
106
106
  return cls(
@@ -140,3 +140,22 @@ def string_to_ideal_type(input_string: str) -> int | bool | float | datetime | s
140
140
  except ValueError:
141
141
  # Return the input string if no conversion is possible
142
142
  return input_string
143
+
144
+
145
+ def get_parameters_by_method(obj: object, prefix: str = "") -> dict[str, dict[str, type]]:
146
+ annotation_by_method = {}
147
+ namespace_dict = {**dict(obj.__class__.__dict__.items()), **vars(obj)}
148
+ for name, value in namespace_dict.items():
149
+ if name != "__call__" and (name.startswith("_") or isinstance(value, property)):
150
+ continue
151
+ if callable(value) and type(value).__name__ == "function":
152
+ annotation_by_method[f"{prefix}{name}"] = get_parameters(value)
153
+ elif isinstance(value, object) and type(value).__module__ != "builtins":
154
+ sub_prefix = f"{prefix}{name}." if prefix else f"{name}."
155
+ annotation_by_method.update(get_parameters_by_method(value, sub_prefix))
156
+ return annotation_by_method
157
+
158
+
159
+ def get_parameters(obj: Callable) -> dict[str, type]:
160
+ annotations = inspect.get_annotations(obj)
161
+ return {name: annotations[name] for name in annotations if name != "return"}
@@ -173,7 +173,7 @@ def get_inheritance_path(child: Any, child_parent: dict[Any, list[Any]]) -> list
173
173
  return path
174
174
 
175
175
 
176
- def add_triples_in_batch(graph: Graph, triples: Iterable[Triple], batch_size: int = 10_000):
176
+ def add_triples_in_batch(graph: Graph, triples: Iterable[Triple], batch_size: int = 10_000) -> None:
177
177
  """Adds triples to the graph store in batches.
178
178
 
179
179
  Args:
@@ -204,3 +204,30 @@ def add_triples_in_batch(graph: Graph, triples: Iterable[Triple], batch_size: in
204
204
  check_commit()
205
205
 
206
206
  check_commit(force_commit=True)
207
+
208
+
209
+ def remove_instance_ids_in_batch(graph: Graph, instance_ids: Iterable[URIRef], batch_size: int = 1_000) -> None:
210
+ """Removes all triples related to the given instances in the graph store in batches.
211
+
212
+ Args:
213
+ graph: The graph store to remove triples from
214
+ instance_ids: list of instances to remove triples from
215
+ batch_size: Batch size of triples per commit, by default 10_000
216
+
217
+ """
218
+ batch_count = 0
219
+
220
+ def check_commit(force_commit: bool = False):
221
+ """Commit nodes to the graph if batch counter is reached or if force_commit is True"""
222
+ nonlocal batch_count
223
+ batch_count += 1
224
+ if force_commit or batch_count >= batch_size:
225
+ graph.commit()
226
+ batch_count = 0
227
+ return
228
+
229
+ for instance_id in instance_ids:
230
+ graph.remove((instance_id, None, None))
231
+ check_commit()
232
+
233
+ check_commit(force_commit=True)
cognite/neat/_version.py CHANGED
@@ -1,2 +1,2 @@
1
- __version__ = "0.97.3"
1
+ __version__ = "0.99.0"
2
2
  __engine__ = "^1.0.3"
@@ -10,9 +10,7 @@ from cognite.client.data_classes import (
10
10
  from cognite.client.data_classes.data_modeling import EdgeApply, NodeApply
11
11
 
12
12
  from cognite.neat._rules.models import (
13
- AssetRules,
14
13
  DMSRules,
15
- DomainRules,
16
14
  InformationRules,
17
15
  )
18
16
  from cognite.neat._store import NeatGraphStore
@@ -20,19 +18,13 @@ from cognite.neat._workflows.steps.step_model import DataContract
20
18
 
21
19
 
22
20
  class MultiRuleData(DataContract):
23
- domain: DomainRules | None = None
24
21
  information: InformationRules | None = None
25
- asset: AssetRules | None = None
26
22
  dms: DMSRules | None = None
27
23
 
28
24
  @classmethod
29
- def from_rules(cls, rules: DomainRules | InformationRules | AssetRules | DMSRules):
30
- if isinstance(rules, DomainRules):
31
- return cls(domain=rules)
32
- elif isinstance(rules, InformationRules):
25
+ def from_rules(cls, rules: InformationRules | DMSRules):
26
+ if isinstance(rules, InformationRules):
33
27
  return cls(information=rules)
34
- elif isinstance(rules, AssetRules):
35
- return cls(asset=rules)
36
28
  elif isinstance(rules, DMSRules):
37
29
  return cls(dms=rules)
38
30
  else:
@@ -2,16 +2,14 @@ import time
2
2
  from pathlib import Path
3
3
  from typing import ClassVar, Literal, cast
4
4
 
5
+ from cognite.neat._client import NeatClient
5
6
  from cognite.neat._issues.errors import WorkflowStepNotInitializedError
6
7
  from cognite.neat._rules import exporters
7
8
  from cognite.neat._rules._shared import DMSRules, InformationRules, VerifiedRules
8
- from cognite.neat._rules.models import AssetRules, RoleTypes
9
+ from cognite.neat._rules.models import RoleTypes
9
10
  from cognite.neat._rules.transformers import (
10
- AssetToInformation,
11
11
  DMSToInformation,
12
- InformationToAsset,
13
12
  InformationToDMS,
14
- RulesPipeline,
15
13
  )
16
14
  from cognite.neat._workflows.model import FlowMessage, StepExecutionStatus
17
15
  from cognite.neat._workflows.steps.data_contracts import CogniteClient, MultiRuleData
@@ -82,7 +80,7 @@ class DeleteDataModelFromCDF(Step):
82
80
  error_text="No DMS Schema components selected for removal! Please select minimum one!",
83
81
  step_execution_status=StepExecutionStatus.ABORT_AND_FAIL,
84
82
  )
85
- input_rules = rules.dms or rules.information or rules.asset
83
+ input_rules = rules.dms or rules.information
86
84
  if input_rules is None:
87
85
  return FlowMessage(
88
86
  error_text="Missing DMS or Information rules in the input data! "
@@ -93,8 +91,6 @@ class DeleteDataModelFromCDF(Step):
93
91
  dms_rules = input_rules
94
92
  elif isinstance(input_rules, InformationRules):
95
93
  dms_rules = InformationToDMS().transform(input_rules).rules
96
- elif isinstance(input_rules, AssetRules):
97
- dms_rules = RulesPipeline[AssetRules, DMSRules]([AssetToInformation(), InformationToDMS()]).run(input_rules)
98
94
  else:
99
95
  raise NotImplementedError(f"Unsupported rules type {type(input_rules)}")
100
96
 
@@ -105,7 +101,7 @@ class DeleteDataModelFromCDF(Step):
105
101
 
106
102
  report_lines = ["# Data Model Deletion from CDF\n\n"]
107
103
  errors = []
108
- for result in dms_exporter.delete_from_cdf(rules=dms_rules, client=cdf_client, dry_run=dry_run):
104
+ for result in dms_exporter.delete_from_cdf(rules=dms_rules, client=NeatClient(cdf_client), dry_run=dry_run):
109
105
  report_lines.append(str(result))
110
106
  errors.extend(result.error_messages)
111
107
 
@@ -207,8 +203,6 @@ class RulesToDMS(Step):
207
203
  dms_rules = input_rules
208
204
  elif isinstance(input_rules, InformationRules):
209
205
  dms_rules = InformationToDMS().transform(input_rules).rules
210
- elif isinstance(input_rules, AssetRules):
211
- dms_rules = RulesPipeline[AssetRules, DMSRules]([AssetToInformation(), InformationToDMS()]).run(input_rules)
212
206
  else:
213
207
  raise NotImplementedError(f"Unsupported rules type {type(input_rules)}")
214
208
 
@@ -220,18 +214,16 @@ class RulesToDMS(Step):
220
214
 
221
215
  output_dir = self.config.staging_path
222
216
  output_dir.mkdir(parents=True, exist_ok=True)
223
- file_name = (
224
- input_rules.metadata.external_id
225
- if isinstance(input_rules, DMSRules)
226
- else input_rules.metadata.name.replace(" ", "_").lower()
227
- )
217
+ file_name = input_rules.metadata.external_id
228
218
  schema_zip = f"{file_name}.zip"
229
219
  schema_full_path = output_dir / schema_zip
230
220
  dms_exporter.export_to_file(dms_rules, schema_full_path)
231
221
 
232
222
  report_lines = ["# DMS Schema Export to CDF\n\n"]
233
223
  errors = []
234
- for result in dms_exporter.export_to_cdf_iterable(rules=dms_rules, client=cdf_client, dry_run=dry_run):
224
+ for result in dms_exporter.export_to_cdf_iterable(
225
+ rules=dms_rules, client=NeatClient(cdf_client), dry_run=dry_run
226
+ ):
235
227
  report_lines.append(str(result))
236
228
  errors.extend(result.error_messages)
237
229
 
@@ -283,14 +275,6 @@ class RulesToExcel(Step):
283
275
  "rules will be used.",
284
276
  options=["input", *RoleTypes.__members__.keys()],
285
277
  ),
286
- Configurable(
287
- name="Dump Format",
288
- value="user",
289
- label="How to dump the rules to the Excel file.\n"
290
- "'user' - just as is.\n'reference' - enterprise model used as basis for a solution model\n"
291
- "'last' - used when updating a data model.",
292
- options=list(exporters.ExcelExporter.dump_options),
293
- ),
294
278
  Configurable(
295
279
  name="New Data Model ID",
296
280
  value="",
@@ -308,7 +292,6 @@ class RulesToExcel(Step):
308
292
  if self.configs is None or self.data_store_path is None:
309
293
  raise WorkflowStepNotInitializedError(type(self).__name__)
310
294
 
311
- dump_format = self.configs.get("Dump Format", "user")
312
295
  styling = cast(exporters.ExcelExporter.Style, self.configs.get("Styling", "default"))
313
296
  role = self.configs.get("Output role format")
314
297
  output_role: RoleTypes | None = None
@@ -325,18 +308,15 @@ class RulesToExcel(Step):
325
308
  step_execution_status=StepExecutionStatus.ABORT_AND_FAIL,
326
309
  )
327
310
 
328
- excel_exporter = exporters.ExcelExporter(styling=styling, dump_as=dump_format, new_model_id=new_model_id) # type: ignore[arg-type]
311
+ excel_exporter = exporters.ExcelExporter(styling=styling, new_model_id=new_model_id) # type: ignore[arg-type]
329
312
 
330
313
  # Todo - Move the conversion to a separate workflow step.
331
314
  rule_instance: VerifiedRules
332
- if rules.domain:
333
- rule_instance = rules.domain
334
- elif rules.information:
315
+
316
+ if rules.information:
335
317
  rule_instance = rules.information
336
318
  elif rules.dms:
337
319
  rule_instance = rules.dms
338
- elif rules.asset:
339
- rule_instance = rules.asset
340
320
  else:
341
321
  output_errors = "No rules provided for export!"
342
322
  return FlowMessage(error_text=output_errors, step_execution_status=StepExecutionStatus.ABORT_AND_FAIL)
@@ -346,26 +326,11 @@ class RulesToExcel(Step):
346
326
  elif output_role is RoleTypes.dms:
347
327
  if isinstance(rule_instance, InformationRules):
348
328
  rule_instance = InformationToDMS().transform(rule_instance).rules
349
- elif isinstance(rule_instance, AssetRules):
350
- rule_instance = RulesPipeline[AssetRules, DMSRules]([AssetToInformation(), InformationToDMS()]).run(
351
- rule_instance
352
- )
353
329
  else:
354
330
  raise NotImplementedError(f"Role {output_role} is not supported for {type(rules).__name__} rules")
355
331
  elif output_role is RoleTypes.information:
356
332
  if isinstance(rule_instance, DMSRules):
357
333
  rule_instance = DMSToInformation().transform(rule_instance).rules
358
- elif isinstance(rule_instance, AssetRules):
359
- rule_instance = AssetToInformation().transform(rule_instance).rules
360
- else:
361
- raise NotImplementedError(f"Role {output_role} is not supported for {type(rules).__name__} rules")
362
- elif output_role is RoleTypes.asset:
363
- if isinstance(rule_instance, InformationRules):
364
- rule_instance = InformationToAsset().transform(rule_instance).rules
365
- elif isinstance(rule_instance, DMSRules):
366
- rule_instance = RulesPipeline[DMSRules, AssetRules]([DMSToInformation(), InformationToAsset()]).run(
367
- rule_instance
368
- )
369
334
  else:
370
335
  raise NotImplementedError(f"Role {output_role} is not supported for {type(rules).__name__} rules")
371
336
  else:
@@ -604,8 +569,6 @@ class RulesToCDFTransformations(Step):
604
569
  dms_rules = input_rules
605
570
  elif isinstance(input_rules, InformationRules):
606
571
  dms_rules = InformationToDMS().transform(input_rules).rules
607
- elif isinstance(input_rules, AssetRules):
608
- dms_rules = RulesPipeline[AssetRules, DMSRules]([AssetToInformation(), InformationToDMS()]).run(input_rules)
609
572
  else:
610
573
  raise NotImplementedError(f"Unsupported rules type {type(input_rules)}")
611
574
 
@@ -624,7 +587,9 @@ class RulesToCDFTransformations(Step):
624
587
 
625
588
  report_lines = ["# DMS Schema Export to CDF\n\n"]
626
589
  errors = []
627
- for result in dms_exporter.export_to_cdf_iterable(rules=dms_rules, client=cdf_client, dry_run=dry_run):
590
+ for result in dms_exporter.export_to_cdf_iterable(
591
+ rules=dms_rules, client=NeatClient(cdf_client), dry_run=dry_run
592
+ ):
628
593
  report_lines.append(str(result))
629
594
  errors.extend(result.error_messages)
630
595
 
@@ -5,6 +5,7 @@ from typing import ClassVar
5
5
  from cognite.client import CogniteClient
6
6
  from cognite.client.data_classes.data_modeling import DataModelId
7
7
 
8
+ from cognite.neat._client import NeatClient
8
9
  from cognite.neat._issues.errors import WorkflowStepNotInitializedError
9
10
  from cognite.neat._issues.formatters import FORMATTER_BY_NAME
10
11
  from cognite.neat._rules import importers
@@ -299,7 +300,9 @@ class DMSToRules(Step):
299
300
  return FlowMessage(error_text=error_text, step_execution_status=StepExecutionStatus.ABORT_AND_FAIL)
300
301
  ref_model_id = ref_model.as_id()
301
302
 
302
- dms_importer = importers.DMSImporter.from_data_model_id(cdf_client, datamodel_entity.as_id(), ref_model_id)
303
+ dms_importer = importers.DMSImporter.from_data_model_id(
304
+ NeatClient(cdf_client), datamodel_entity.as_id(), ref_model_id
305
+ )
303
306
 
304
307
  # if role is None, it will be inferred from the rules file
305
308
  role = self.configs.get("Role")
@@ -5,11 +5,12 @@ from typing import ClassVar
5
5
 
6
6
  from cognite.client import CogniteClient
7
7
 
8
+ from cognite.neat._client import NeatClient
9
+ from cognite.neat._client._api.data_modeling_loaders import ViewLoader
8
10
  from cognite.neat._issues import NeatIssueList
9
11
  from cognite.neat._issues.errors import ResourceNotFoundError, WorkflowStepNotInitializedError
10
12
  from cognite.neat._issues.formatters import FORMATTER_BY_NAME
11
- from cognite.neat._rules.models import DMSRules, SchemaCompleteness
12
- from cognite.neat._utils.cdf.loaders import ViewLoader
13
+ from cognite.neat._rules.models import DMSRules
13
14
  from cognite.neat._workflows.model import FlowMessage, StepExecutionStatus
14
15
  from cognite.neat._workflows.steps.data_contracts import MultiRuleData
15
16
  from cognite.neat._workflows.steps.step_model import Configurable, Step
@@ -49,12 +50,7 @@ class ValidateRulesAgainstCDF(Step):
49
50
  step_execution_status=StepExecutionStatus.ABORT_AND_FAIL,
50
51
  )
51
52
  dms_rules = rules.dms
52
- if dms_rules.metadata.schema_ is not SchemaCompleteness.partial:
53
- return FlowMessage(
54
- error_text="DMS rules are not partial. This step expects DMS rules to be a partial definition "
55
- "with the rest of the definition being fetched from CDF.",
56
- step_execution_status=StepExecutionStatus.ABORT_AND_FAIL,
57
- )
53
+
58
54
  schema = dms_rules.as_schema()
59
55
  errors = schema.validate()
60
56
  if not errors:
@@ -80,7 +76,7 @@ class ValidateRulesAgainstCDF(Step):
80
76
  retrieved_containers = cdf_client.data_modeling.containers.retrieve(missing_containers).as_write()
81
77
  # Converting read format of views to write format requires to account for parents (implements)
82
78
  # Thus we use the loader to convert the views to write format.
83
- view_loader = ViewLoader(cdf_client)
79
+ view_loader = ViewLoader(NeatClient(cdf_client))
84
80
  retrieved_views = [
85
81
  view_loader.as_write(view) for view in cdf_client.data_modeling.views.retrieve(missing_views)
86
82
  ]
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: cognite-neat
3
- Version: 0.97.3
3
+ Version: 0.99.0
4
4
  Summary: Knowledge graph transformation
5
5
  Home-page: https://cognite-neat.readthedocs-hosted.com/
6
6
  License: Apache-2.0
@@ -29,6 +29,7 @@ Requires-Dist: google-auth-oauthlib ; extra == "google"
29
29
  Requires-Dist: gspread ; extra == "google"
30
30
  Requires-Dist: jsonpath-python (>=1.0.6,<2.0.0)
31
31
  Requires-Dist: lxml (>=5.3.0,<6.0.0) ; extra == "all"
32
+ Requires-Dist: mixpanel (>=4.10.1,<5.0.0)
32
33
  Requires-Dist: mkdocs ; extra == "docs"
33
34
  Requires-Dist: mkdocs-autorefs (>=0.5.0,<0.6.0) ; extra == "docs"
34
35
  Requires-Dist: mkdocs-git-authors-plugin ; extra == "docs"
@@ -40,13 +41,13 @@ Requires-Dist: mkdocs-material-extensions ; extra == "docs"
40
41
  Requires-Dist: mkdocstrings[python] ; extra == "docs"
41
42
  Requires-Dist: networkx (>=3.4.2,<4.0.0)
42
43
  Requires-Dist: openpyxl
43
- Requires-Dist: oxrdflib[oxigraph] (>=0.3.3,<0.4.0) ; extra == "oxi" or extra == "all"
44
+ Requires-Dist: oxrdflib[oxigraph] (>=0.4.0,<0.5.0) ; extra == "oxi" or extra == "all"
44
45
  Requires-Dist: packaging (>=22.0,<25.0)
45
46
  Requires-Dist: pandas
46
47
  Requires-Dist: prometheus-client (>=0,<1) ; extra == "service" or extra == "all"
47
48
  Requires-Dist: pydantic (>=2,<3)
48
49
  Requires-Dist: pymdown-extensions ; extra == "docs"
49
- Requires-Dist: pyoxigraph (==0.3.19) ; extra == "oxi" or extra == "all"
50
+ Requires-Dist: pyoxigraph (==0.4.3) ; extra == "oxi" or extra == "all"
50
51
  Requires-Dist: python-multipart (==0.0.9) ; extra == "service" or extra == "all"
51
52
  Requires-Dist: pyvis (>=0.3.2,<0.4.0)
52
53
  Requires-Dist: rdflib