cognite-neat 0.87.4__py3-none-any.whl → 0.88.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cognite-neat might be problematic. Click here for more details.

Files changed (132) hide show
  1. cognite/neat/_version.py +1 -1
  2. cognite/neat/app/api/data_classes/rest.py +0 -19
  3. cognite/neat/app/api/explorer.py +6 -4
  4. cognite/neat/app/api/routers/crud.py +11 -21
  5. cognite/neat/app/api/routers/workflows.py +24 -94
  6. cognite/neat/graph/extractors/_classic_cdf/_assets.py +8 -2
  7. cognite/neat/graph/extractors/_mock_graph_generator.py +2 -2
  8. cognite/neat/graph/loaders/_base.py +17 -12
  9. cognite/neat/graph/loaders/_rdf2asset.py +223 -58
  10. cognite/neat/graph/loaders/_rdf2dms.py +1 -1
  11. cognite/neat/graph/stores/_base.py +5 -0
  12. cognite/neat/rules/analysis/_asset.py +31 -1
  13. cognite/neat/rules/importers/_inference2rules.py +31 -35
  14. cognite/neat/rules/models/information/_rules.py +1 -1
  15. cognite/neat/workflows/steps/data_contracts.py +17 -43
  16. cognite/neat/workflows/steps/lib/current/graph_extractor.py +28 -24
  17. cognite/neat/workflows/steps/lib/current/graph_loader.py +4 -21
  18. cognite/neat/workflows/steps/lib/current/graph_store.py +18 -134
  19. cognite/neat/workflows/steps_registry.py +5 -7
  20. {cognite_neat-0.87.4.dist-info → cognite_neat-0.88.0.dist-info}/METADATA +1 -1
  21. {cognite_neat-0.87.4.dist-info → cognite_neat-0.88.0.dist-info}/RECORD +24 -132
  22. cognite/neat/app/api/routers/core.py +0 -91
  23. cognite/neat/app/api/routers/data_exploration.py +0 -336
  24. cognite/neat/app/api/routers/rules.py +0 -203
  25. cognite/neat/legacy/__init__.py +0 -0
  26. cognite/neat/legacy/graph/__init__.py +0 -3
  27. cognite/neat/legacy/graph/examples/Knowledge-Graph-Nordic44-dirty.xml +0 -20182
  28. cognite/neat/legacy/graph/examples/Knowledge-Graph-Nordic44.xml +0 -20163
  29. cognite/neat/legacy/graph/examples/__init__.py +0 -10
  30. cognite/neat/legacy/graph/examples/skos-capturing-sheet-wind-topics.xlsx +0 -0
  31. cognite/neat/legacy/graph/exceptions.py +0 -90
  32. cognite/neat/legacy/graph/extractors/__init__.py +0 -6
  33. cognite/neat/legacy/graph/extractors/_base.py +0 -14
  34. cognite/neat/legacy/graph/extractors/_dexpi.py +0 -44
  35. cognite/neat/legacy/graph/extractors/_graph_capturing_sheet.py +0 -403
  36. cognite/neat/legacy/graph/extractors/_mock_graph_generator.py +0 -361
  37. cognite/neat/legacy/graph/loaders/__init__.py +0 -23
  38. cognite/neat/legacy/graph/loaders/_asset_loader.py +0 -511
  39. cognite/neat/legacy/graph/loaders/_base.py +0 -67
  40. cognite/neat/legacy/graph/loaders/_exceptions.py +0 -85
  41. cognite/neat/legacy/graph/loaders/core/__init__.py +0 -0
  42. cognite/neat/legacy/graph/loaders/core/labels.py +0 -58
  43. cognite/neat/legacy/graph/loaders/core/models.py +0 -136
  44. cognite/neat/legacy/graph/loaders/core/rdf_to_assets.py +0 -1046
  45. cognite/neat/legacy/graph/loaders/core/rdf_to_relationships.py +0 -559
  46. cognite/neat/legacy/graph/loaders/rdf_to_dms.py +0 -309
  47. cognite/neat/legacy/graph/loaders/validator.py +0 -87
  48. cognite/neat/legacy/graph/models.py +0 -6
  49. cognite/neat/legacy/graph/stores/__init__.py +0 -13
  50. cognite/neat/legacy/graph/stores/_base.py +0 -400
  51. cognite/neat/legacy/graph/stores/_graphdb_store.py +0 -52
  52. cognite/neat/legacy/graph/stores/_memory_store.py +0 -43
  53. cognite/neat/legacy/graph/stores/_oxigraph_store.py +0 -151
  54. cognite/neat/legacy/graph/stores/_oxrdflib.py +0 -247
  55. cognite/neat/legacy/graph/stores/_rdf_to_graph.py +0 -42
  56. cognite/neat/legacy/graph/transformations/__init__.py +0 -0
  57. cognite/neat/legacy/graph/transformations/entity_matcher.py +0 -101
  58. cognite/neat/legacy/graph/transformations/query_generator/__init__.py +0 -3
  59. cognite/neat/legacy/graph/transformations/query_generator/sparql.py +0 -575
  60. cognite/neat/legacy/graph/transformations/transformer.py +0 -322
  61. cognite/neat/legacy/rules/__init__.py +0 -0
  62. cognite/neat/legacy/rules/analysis.py +0 -231
  63. cognite/neat/legacy/rules/examples/Rules-Nordic44-to-graphql.xlsx +0 -0
  64. cognite/neat/legacy/rules/examples/Rules-Nordic44.xlsx +0 -0
  65. cognite/neat/legacy/rules/examples/__init__.py +0 -18
  66. cognite/neat/legacy/rules/examples/power-grid-containers.yaml +0 -124
  67. cognite/neat/legacy/rules/examples/power-grid-example.xlsx +0 -0
  68. cognite/neat/legacy/rules/examples/power-grid-model.yaml +0 -224
  69. cognite/neat/legacy/rules/examples/rules-template.xlsx +0 -0
  70. cognite/neat/legacy/rules/examples/sheet2cdf-transformation-rules.xlsx +0 -0
  71. cognite/neat/legacy/rules/examples/skos-rules.xlsx +0 -0
  72. cognite/neat/legacy/rules/examples/source-to-solution-mapping-rules.xlsx +0 -0
  73. cognite/neat/legacy/rules/examples/wind-energy.owl +0 -1511
  74. cognite/neat/legacy/rules/exceptions.py +0 -2972
  75. cognite/neat/legacy/rules/exporters/__init__.py +0 -20
  76. cognite/neat/legacy/rules/exporters/_base.py +0 -45
  77. cognite/neat/legacy/rules/exporters/_core/__init__.py +0 -5
  78. cognite/neat/legacy/rules/exporters/_core/rules2labels.py +0 -24
  79. cognite/neat/legacy/rules/exporters/_rules2dms.py +0 -885
  80. cognite/neat/legacy/rules/exporters/_rules2excel.py +0 -213
  81. cognite/neat/legacy/rules/exporters/_rules2graphql.py +0 -183
  82. cognite/neat/legacy/rules/exporters/_rules2ontology.py +0 -524
  83. cognite/neat/legacy/rules/exporters/_rules2pydantic_models.py +0 -748
  84. cognite/neat/legacy/rules/exporters/_rules2rules.py +0 -105
  85. cognite/neat/legacy/rules/exporters/_rules2triples.py +0 -38
  86. cognite/neat/legacy/rules/exporters/_validation.py +0 -146
  87. cognite/neat/legacy/rules/importers/__init__.py +0 -22
  88. cognite/neat/legacy/rules/importers/_base.py +0 -66
  89. cognite/neat/legacy/rules/importers/_dict2rules.py +0 -158
  90. cognite/neat/legacy/rules/importers/_dms2rules.py +0 -194
  91. cognite/neat/legacy/rules/importers/_graph2rules.py +0 -308
  92. cognite/neat/legacy/rules/importers/_json2rules.py +0 -39
  93. cognite/neat/legacy/rules/importers/_owl2rules/__init__.py +0 -3
  94. cognite/neat/legacy/rules/importers/_owl2rules/_owl2classes.py +0 -239
  95. cognite/neat/legacy/rules/importers/_owl2rules/_owl2metadata.py +0 -260
  96. cognite/neat/legacy/rules/importers/_owl2rules/_owl2properties.py +0 -217
  97. cognite/neat/legacy/rules/importers/_owl2rules/_owl2rules.py +0 -290
  98. cognite/neat/legacy/rules/importers/_spreadsheet2rules.py +0 -45
  99. cognite/neat/legacy/rules/importers/_xsd2rules.py +0 -20
  100. cognite/neat/legacy/rules/importers/_yaml2rules.py +0 -39
  101. cognite/neat/legacy/rules/models/__init__.py +0 -5
  102. cognite/neat/legacy/rules/models/_base.py +0 -151
  103. cognite/neat/legacy/rules/models/raw_rules.py +0 -316
  104. cognite/neat/legacy/rules/models/rdfpath.py +0 -237
  105. cognite/neat/legacy/rules/models/rules.py +0 -1289
  106. cognite/neat/legacy/rules/models/tables.py +0 -9
  107. cognite/neat/legacy/rules/models/value_types.py +0 -118
  108. cognite/neat/legacy/workflows/examples/Export_DMS/workflow.yaml +0 -89
  109. cognite/neat/legacy/workflows/examples/Export_Rules_to_Ontology/workflow.yaml +0 -152
  110. cognite/neat/legacy/workflows/examples/Extract_DEXPI_Graph_and_Export_Rules/workflow.yaml +0 -139
  111. cognite/neat/legacy/workflows/examples/Extract_RDF_Graph_and_Generate_Assets/workflow.yaml +0 -270
  112. cognite/neat/legacy/workflows/examples/Import_DMS/workflow.yaml +0 -65
  113. cognite/neat/legacy/workflows/examples/Ontology_to_Data_Model/workflow.yaml +0 -116
  114. cognite/neat/legacy/workflows/examples/Validate_Rules/workflow.yaml +0 -67
  115. cognite/neat/legacy/workflows/examples/Validate_Solution_Model/workflow.yaml +0 -64
  116. cognite/neat/legacy/workflows/examples/Visualize_Data_Model_Using_Mock_Graph/workflow.yaml +0 -95
  117. cognite/neat/legacy/workflows/examples/Visualize_Semantic_Data_Model/workflow.yaml +0 -111
  118. cognite/neat/workflows/examples/Extract_RDF_Graph_and_Generate_Assets/workflow.yaml +0 -270
  119. cognite/neat/workflows/migration/__init__.py +0 -0
  120. cognite/neat/workflows/migration/steps.py +0 -91
  121. cognite/neat/workflows/migration/wf_manifests.py +0 -33
  122. cognite/neat/workflows/steps/lib/legacy/__init__.py +0 -7
  123. cognite/neat/workflows/steps/lib/legacy/graph_contextualization.py +0 -82
  124. cognite/neat/workflows/steps/lib/legacy/graph_extractor.py +0 -746
  125. cognite/neat/workflows/steps/lib/legacy/graph_loader.py +0 -606
  126. cognite/neat/workflows/steps/lib/legacy/graph_store.py +0 -307
  127. cognite/neat/workflows/steps/lib/legacy/graph_transformer.py +0 -58
  128. cognite/neat/workflows/steps/lib/legacy/rules_exporter.py +0 -511
  129. cognite/neat/workflows/steps/lib/legacy/rules_importer.py +0 -612
  130. {cognite_neat-0.87.4.dist-info → cognite_neat-0.88.0.dist-info}/LICENSE +0 -0
  131. {cognite_neat-0.87.4.dist-info → cognite_neat-0.88.0.dist-info}/WHEEL +0 -0
  132. {cognite_neat-0.87.4.dist-info → cognite_neat-0.88.0.dist-info}/entry_points.txt +0 -0
@@ -1,748 +0,0 @@
1
- import re
2
- import sys
3
- import warnings
4
- from collections.abc import Iterable
5
- from datetime import date, datetime
6
- from typing import Any, TypeAlias, cast
7
-
8
- from cognite.client.data_classes import Asset, Relationship
9
- from cognite.client.data_classes.data_modeling import EdgeApply, MappedPropertyApply, NodeApply, NodeOrEdgeData, ViewId
10
- from cognite.client.data_classes.data_modeling.views import SingleHopConnectionDefinitionApply, ViewApply
11
- from pydantic import BaseModel, ConfigDict, Field, create_model
12
- from pydantic._internal._model_construction import ModelMetaclass
13
- from rdflib import Graph, URIRef
14
-
15
- from cognite.neat.legacy.graph.loaders.core.rdf_to_assets import NeatMetadataKeys
16
- from cognite.neat.legacy.graph.transformations.query_generator.sparql import build_construct_query, triples2dictionary
17
- from cognite.neat.legacy.rules import exceptions
18
- from cognite.neat.legacy.rules.analysis import define_class_asset_mapping, to_class_property_pairs
19
- from cognite.neat.legacy.rules.exporters._rules2dms import DMSSchemaComponents
20
- from cognite.neat.legacy.rules.exporters._validation import are_entity_names_dms_compliant
21
- from cognite.neat.legacy.rules.models.rules import Property, Rules
22
- from cognite.neat.legacy.rules.models.value_types import ValueTypeMapping
23
- from cognite.neat.utils.auxiliary import create_sha256_hash, generate_exception_report
24
-
25
- if sys.version_info >= (3, 11):
26
- from datetime import UTC
27
- else:
28
- from datetime import timezone
29
-
30
- UTC = timezone.utc
31
-
32
- EdgeOneToOne: TypeAlias = str # type: ignore[valid-type]
33
- EdgeOneToMany: TypeAlias = list[str] # type: ignore[valid-type]
34
-
35
-
36
- def default_model_configuration(
37
- external_id: str | None = None,
38
- name: str | None = None,
39
- description: str | None = None,
40
- space: str | None = None,
41
- version: str | None = None,
42
- ) -> ConfigDict:
43
- return ConfigDict(
44
- populate_by_name=True,
45
- str_strip_whitespace=True,
46
- arbitrary_types_allowed=True,
47
- strict=False,
48
- extra="allow",
49
- json_schema_extra={
50
- "title": name,
51
- "description": description,
52
- "external_id": external_id,
53
- "name": name,
54
- "space": space,
55
- "version": version,
56
- },
57
- )
58
-
59
-
60
- def default_class_methods():
61
- return [
62
- from_dict,
63
- from_graph,
64
- to_asset,
65
- to_relationship,
66
- to_node,
67
- to_edge,
68
- get_field_description,
69
- get_field_name,
70
- ]
71
-
72
-
73
- def default_class_property_methods():
74
- return [model_name, model_external_id, model_description, attributes, edges_one_to_one, edges_one_to_many]
75
-
76
-
77
- def rules_to_pydantic_models(
78
- rules: Rules,
79
- methods: list | None = None,
80
- add_extra_fields: bool = False,
81
- ) -> dict[str, ModelMetaclass]:
82
- """
83
- Generate pydantic models from rules.
84
-
85
- Args:
86
- rules: Rules to generate pydantic models from
87
- methods: List of methods to register for pydantic models,
88
- by default None meaning defaulting to base neat methods.
89
- add_extra_fields: Flag indicating to add extra fields to pydantic models, by default False
90
-
91
- Returns:
92
- Dictionary containing pydantic models with class ids as key and pydantic model as value
93
- containing properties defined for the given class(es) in the rules.
94
-
95
-
96
- !!! note "Default NEAT methods"
97
- Default NEAT methods which are added to the generated pydantic models are:
98
-
99
- - `get_field_description`: Returns description of the field if one exists.
100
- - `get_field_name`: Returns name of the field if one exists.
101
- - `model_name`: Returns the name of the model if one exists.
102
- - `model_external_id`: Returns the external id of the model if one exists.
103
- - `model_description`: Returns the description of the model if one exists.
104
- - `from_graph`: Creates model instance from class instance stored in RDF graph.
105
- - `to_asset`: Creates CDF Asset instance from model instance.
106
- - `to_node`: Creates DMS node from model instance.
107
- - `to_edge`: Creates DMS edge from model instance.
108
- - `attributes`: Returns list of node attributes.
109
- - `edges_one_to_one`: Returns list of node edges one to one.
110
- - `edges_one_to_many`: Returns list of node edges one to many.
111
-
112
-
113
- !!! note "Limitations"
114
- Currently this will take only unique properties and those which column rule_type
115
- is set to rdfpath, hence only_rdfpath = True. This means that at the moment
116
- we do not support UNION, i.e. ability to handle multiple rdfpaths for the same
117
- property. This is needed option and should be added in the second version of the exporter.
118
-
119
- !!! note "Classes and Properties must be DMS Compliant"
120
- Rules must be DMS compliant, i.e. class and property ids must obey the following regex:
121
- r`(^[a-zA-Z][a-zA-Z0-9_]{0,253}[a-zA-Z0-9]?$)` and must not contain reserved keywords.
122
-
123
- Classes reserved keywords: `Query`, `Mutation`, `Subscription`, `String`, `Int32`,
124
- `Int64`, `Int`, `Float32`, `Float64`, `Float`,`Timestamp`, `JSONObject`, `Date`,
125
- `Numeric`, `Boolean`, `PageInfo`, `File`, `Sequence`, `TimeSeries`
126
-
127
- Properties reserved keywords: `space`, `externalId`, `createdTime`, `lastUpdatedTime`,
128
- `deletedTime`, `edge_id` `node_id`, `project_id`, `property_group`, `seq`, `tg_table_name`, `extensions`
129
- """
130
-
131
- names_compliant, name_warnings = are_entity_names_dms_compliant(rules, return_report=True)
132
- if not names_compliant:
133
- raise exceptions.EntitiesContainNonDMSCompliantCharacters(report=generate_exception_report(name_warnings))
134
-
135
- if methods is None:
136
- methods = default_class_methods() + default_class_property_methods()
137
-
138
- class_property_pairs = to_class_property_pairs(rules, only_rdfpath=True)
139
-
140
- models: dict[str, ModelMetaclass] = {}
141
- for class_, properties in class_property_pairs.items():
142
- # generate fields from define properties
143
- fields = _properties_to_pydantic_fields(properties)
144
-
145
- if add_extra_fields:
146
- # store default class to relationship mapping field
147
- # which is used by the `to_relationship` method
148
- fields["class_to_asset_mapping"] = (
149
- dict[str, list[str]],
150
- Field(
151
- define_class_asset_mapping(rules, class_),
152
- description="This is a helper field used for generating CDF Asset out of model instance",
153
- ),
154
- )
155
-
156
- model = _dictionary_to_pydantic_model(
157
- model_id=class_,
158
- model_fields_definition=fields,
159
- model_name=rules.classes[class_].class_name,
160
- model_description=rules.classes[class_].description,
161
- model_methods=methods,
162
- space=rules.metadata.prefix,
163
- version=rules.metadata.version,
164
- )
165
-
166
- models[class_] = model
167
-
168
- return models
169
-
170
-
171
- def _properties_to_pydantic_fields(
172
- properties: dict[str, Property],
173
- ) -> dict[str, tuple[EdgeOneToMany | EdgeOneToOne | type | list[type], Any]]:
174
- """Turns definition of properties into pydantic fields.
175
-
176
- Parameters
177
- ----------
178
- properties : dict[str, Property]
179
- Dictionary of properties
180
-
181
- Returns
182
- -------
183
- dict[str, tuple[EdgeOneToMany | EdgeOneToOne | type | list[type], Any]]
184
- Dictionary of pydantic fields
185
- """
186
-
187
- fields: dict[str, tuple[EdgeOneToMany | EdgeOneToOne | type | list[type], Any]]
188
-
189
- fields = {"external_id": (str, Field(..., alias="external_id"))}
190
-
191
- for property_id, property_ in properties.items():
192
- field_type = _define_field_type(property_)
193
- field_definition: dict = {
194
- "alias": property_.property_name,
195
- "description": property_.description if property_.description else None,
196
- # keys below will be available under json_schema_extra
197
- "property_type": field_type.__name__ if field_type in [EdgeOneToOne, EdgeOneToMany] else "NodeAttribute",
198
- "property_value_type": property_.expected_value_type.suffix,
199
- "property_name": property_.property_name,
200
- "property_id": property_.property_id,
201
- }
202
-
203
- if field_type.__name__ in [EdgeOneToMany.__name__, list.__name__]:
204
- field_definition["min_length"] = property_.min_count
205
- field_definition["max_length"] = property_.max_count
206
-
207
- if not property_.is_mandatory and not property_.default:
208
- field_definition["default"] = None
209
- elif property_.default:
210
- field_definition["default"] = property_.default
211
-
212
- fields[property_id] = (
213
- field_type,
214
- Field(**field_definition), # type: ignore[pydantic-field]
215
- )
216
- return fields
217
-
218
-
219
- def _define_field_type(property_: Property):
220
- if property_.property_type == "ObjectProperty" and property_.max_count == 1:
221
- return EdgeOneToOne
222
- elif property_.property_type == "ObjectProperty":
223
- return EdgeOneToMany
224
- elif property_.property_type == "DatatypeProperty" and property_.max_count == 1:
225
- return cast(ValueTypeMapping, property_.expected_value_type.mapping).python
226
- else:
227
- inner_type = cast(ValueTypeMapping, property_.expected_value_type.mapping).python
228
- return list[inner_type] # type: ignore[valid-type]
229
-
230
-
231
- def _dictionary_to_pydantic_model(
232
- model_id: str,
233
- model_fields_definition: dict,
234
- space: str | None = None,
235
- version: str | None = None,
236
- model_name: str | None = None,
237
- model_description: str | None = None,
238
- model_configuration: ConfigDict | None = None,
239
- model_methods: list | None = None,
240
- validators: list | None = None,
241
- ) -> type[BaseModel]:
242
- """Generates pydantic model from dictionary containing definition of fields.
243
- Additionally, it adds methods to the model and validators.
244
-
245
- Parameters
246
- ----------
247
- model_name : str
248
- Name of the model, typically an id of the class
249
- model_fields_definition : dict
250
- Dictionary containing definition of fields
251
- model_configuration : ConfigDict, optional
252
- Configuration of pydantic model, by default None
253
- methods : list, optional
254
- Methods that work on fields once model is instantiated, by default None
255
- property_attributes : list, optional
256
- Property attributed that work on model fields once model is instantiated, by default None
257
- validators : list, optional
258
- Any custom validators to be added in addition to base pydantic ones, by default None
259
-
260
- Returns
261
- -------
262
- type[BaseModel]
263
- Pydantic model
264
- """
265
-
266
- if not model_configuration:
267
- model_configuration = default_model_configuration(
268
- external_id=model_id, space=space, version=version, name=model_name, description=model_description
269
- )
270
-
271
- fields: dict[str, tuple | type[BaseModel]] = {}
272
-
273
- for field_name, value in model_fields_definition.items():
274
- if isinstance(value, tuple):
275
- fields[field_name] = value
276
- # Nested classes
277
- elif isinstance(value, dict):
278
- fields[field_name] = (_dictionary_to_pydantic_model(f"{model_id}_{field_name}", value), ...)
279
- else:
280
- raise exceptions.FieldValueOfUnknownType(field_name, value)
281
-
282
- model = create_model(model_id, __config__=model_configuration, **fields) # type: ignore[call-overload]
283
-
284
- if model_methods:
285
- for method in model_methods:
286
- try:
287
- setattr(model, method.__name__, method)
288
- except AttributeError:
289
- try:
290
- setattr(model, method.fget.__name__, method)
291
- except AttributeError:
292
- setattr(model, method.__func__.fget.__name__, method)
293
-
294
- # any additional validators to be added
295
- if validators:
296
- ...
297
-
298
- return model
299
-
300
-
301
- @classmethod # type: ignore
302
- @property
303
- def attributes(cls) -> list[str]:
304
- return [
305
- field
306
- for field in cls.model_fields
307
- if (schema := cls.model_fields[field].json_schema_extra) and schema.get("property_type") == "NodeAttribute"
308
- ]
309
-
310
-
311
- @classmethod # type: ignore
312
- @property
313
- def edges_one_to_one(cls) -> list[str]:
314
- return [
315
- field
316
- for field in cls.model_fields
317
- if (schema := cls.model_fields[field].json_schema_extra) and schema.get("property_type") == "EdgeOneToOne"
318
- ]
319
-
320
-
321
- @classmethod # type: ignore
322
- @property
323
- def edges_one_to_many(cls) -> list[str]:
324
- return [
325
- field
326
- for field in cls.model_fields
327
- if (schema := cls.model_fields[field].json_schema_extra) and schema.get("property_type") == "EdgeOneToMany"
328
- ]
329
-
330
-
331
- # Define methods that work on model instance
332
- @classmethod # type: ignore[misc]
333
- def from_dict(cls, dictionary: dict[str, list[str] | str]):
334
- # wrangle results to dict
335
- args: dict[str, list[Any] | Any] = {}
336
- for field in cls.model_fields.values():
337
- # if field is not required and not in result, skip
338
- if not field.is_required() and field.alias not in dictionary:
339
- continue
340
-
341
- # if field is required and not in result, raise error
342
- if field.is_required() and field.alias not in dictionary:
343
- raise exceptions.PropertyRequiredButNotProvided(field.alias, cast(str, dictionary["external_id"]))
344
-
345
- # flatten result if field is not edge or list of values
346
- if field.annotation.__name__ not in [EdgeOneToMany.__name__, list.__name__]:
347
- if isinstance(dictionary[field.alias], list) and len(dictionary[field.alias]) > 1:
348
- warnings.warn(
349
- exceptions.FieldContainsMoreThanOneValue(
350
- field.alias,
351
- len(dictionary[field.alias]),
352
- ).message,
353
- category=exceptions.FieldContainsMoreThanOneValue,
354
- stacklevel=2,
355
- )
356
-
357
- args[field.alias] = dictionary[field.alias][0]
358
- else:
359
- args[field.alias] = dictionary[field.alias]
360
-
361
- return cls(**args)
362
-
363
-
364
- # Define methods that work on model instance
365
- @classmethod # type: ignore[misc]
366
- def from_graph(
367
- cls,
368
- graph: Graph,
369
- transformation_rules: Rules,
370
- external_id: URIRef,
371
- incomplete_instance_allowed: bool = True,
372
- ):
373
- """Method that creates model instance from class instance stored in graph.
374
-
375
- Args:
376
- graph: Graph containing triples of class instance
377
- transformation_rules: Transformation rules
378
- external_id: External id of class instance to be used to instantiate associated pydantic model
379
- incomplete_instance_allowed: Flag allowing incomplete instances to be queried. Defaults to True.
380
-
381
- Raises:
382
- exceptions.MissingInstanceTriples: _description_
383
- exceptions.PropertyRequiredButNotProvided: _description_
384
-
385
- Returns:
386
- Pydantic model instance
387
- """
388
-
389
- # build sparql query for given object
390
- class_ = cls.__name__
391
-
392
- # here properties_optional is set to True in order to also return
393
- # incomplete class instances so we catch them and raise exceptions
394
- sparql_construct_query = build_construct_query(
395
- graph,
396
- class_,
397
- transformation_rules,
398
- class_instances=[external_id],
399
- properties_optional=incomplete_instance_allowed,
400
- )
401
- # In the docs, a construct query is said to return triple
402
- # Not sure if the triple will be URIRef or Literal
403
- query_result = cast(Iterable[tuple[URIRef, URIRef, str | URIRef]], graph.query(sparql_construct_query))
404
-
405
- dictionary = triples2dictionary(query_result)[external_id]
406
-
407
- if not dictionary:
408
- raise exceptions.MissingInstanceTriples(external_id)
409
-
410
- return cls.from_dict(dictionary)
411
-
412
-
413
- # define methods that creates asset out of model id (default)
414
- def to_asset(
415
- self,
416
- data_set_id: int,
417
- add_system_metadata: bool = True,
418
- metadata_keys: NeatMetadataKeys | None = None,
419
- add_labels: bool = True,
420
- ) -> Asset:
421
- """Convert model instance to asset.
422
-
423
- Parameters
424
- ----------
425
- add_system_metadata : bool, optional
426
- Flag indicating to add or not system/neat metadata, by default True
427
- metadata_keys : NeatMetadataKeys | None, optional
428
- Definition of system/neat metadata, by default None
429
- add_labels : bool, optional
430
- To add or not labels to asset, by default True
431
- data_set_id : int, optional
432
- Data set id to which asset belongs to, by default None
433
-
434
- Returns
435
- -------
436
- Asset
437
- Asset instance
438
- """
439
- # Needs copy otherwise modifications impact all instances
440
- if not self.class_to_asset_mapping:
441
- raise exceptions.ClassToAssetMappingNotDefined(self.__class__.__name__)
442
-
443
- class_instance_dictionary = self.model_dump(by_alias=True)
444
- adapted_mapping_config = _adapt_mapping_config_by_instance(
445
- self.external_id, class_instance_dictionary, self.class_to_asset_mapping
446
- )
447
- asset = _class_to_asset_instance_dictionary(class_instance_dictionary, adapted_mapping_config)
448
-
449
- # set default metadata keys if not provided
450
- metadata_keys = NeatMetadataKeys() if metadata_keys is None else metadata_keys
451
-
452
- # add system metadata
453
- if add_system_metadata:
454
- _add_system_metadata(self, metadata_keys, asset)
455
-
456
- if add_labels:
457
- asset["labels"] = [asset["metadata"][metadata_keys.type], "non-historic"]
458
-
459
- return Asset(**asset, data_set_id=data_set_id)
460
-
461
-
462
- def _add_system_metadata(self, metadata_keys: NeatMetadataKeys, asset: dict):
463
- asset["metadata"][metadata_keys.type] = self.__class__.__name__
464
- asset["metadata"][metadata_keys.identifier] = self.external_id
465
- now = str(datetime.now(UTC))
466
- asset["metadata"][metadata_keys.start_time] = now
467
- asset["metadata"][metadata_keys.update_time] = now
468
- asset["metadata"][metadata_keys.active] = "true"
469
-
470
-
471
- def _adapt_mapping_config_by_instance(external_id, class_instance_dictionary, mapping_config):
472
- adapted_mapping_config = {}
473
- for asset_field, class_properties in mapping_config.items():
474
- if asset_field != "metadata":
475
- # We are selecting first property that is available in the graph
476
- # and add it to corresponding asset field and exit loop
477
- for property_ in class_properties:
478
- if class_instance_dictionary.get(property_, None):
479
- adapted_mapping_config[asset_field] = property_
480
- break
481
-
482
- elif metadata_keys := [
483
- property_ for property_ in class_properties if class_instance_dictionary.get(property_, None)
484
- ]:
485
- adapted_mapping_config["metadata"] = metadata_keys
486
-
487
- # Raise warnings for fields that will miss in asset
488
- for asset_field in mapping_config:
489
- if asset_field not in adapted_mapping_config:
490
- warnings.warn(
491
- exceptions.FieldNotFoundInInstance(external_id, asset_field).message,
492
- category=exceptions.FieldNotFoundInInstance,
493
- stacklevel=2,
494
- )
495
-
496
- return adapted_mapping_config
497
-
498
-
499
- def _class_to_asset_instance_dictionary(class_instance_dictionary, mapping_config):
500
- for key, values in mapping_config.items():
501
- if key != "metadata":
502
- mapping_config[key] = class_instance_dictionary.get(values, None)
503
- else:
504
- mapping_config[key] = {value: class_instance_dictionary.get(value, None) for value in values}
505
-
506
- return mapping_config
507
-
508
-
509
- def to_relationship(self, transformation_rules: Rules) -> Relationship:
510
- """Creates relationship instance from model instance."""
511
- raise NotImplementedError()
512
-
513
-
514
- def to_node(
515
- self, data_model_or_view_id: DMSSchemaComponents | ViewId | None = None, add_class_prefix: bool = False
516
- ) -> NodeApply:
517
- """Creates DMS node from the instance of pydantic model.
518
-
519
- Args:
520
- data_model_or_view_id: Instance of DataModel or ViewID. Defaults to None.
521
- add_class_prefix: Whether to add class id (i.e.model name) prefix to external_id of View. Defaults to False.
522
-
523
- Returns:
524
- Instance of NodeApply containing node information.
525
-
526
-
527
- !!! note "Default Behavior"
528
- If no DataModel or ViewID is passed, then the default behavior is to create node
529
- using View information which is by default stored under `model_json_schema` attribute of pydantic model.
530
- !!! note "Limitations"
531
- Currently adding class prefix is only possible if the node is created if ViewID is passed or
532
- if pydantic model already contains View information (which default behavior).
533
- """
534
-
535
- if isinstance(data_model_or_view_id, DMSSchemaComponents):
536
- return _to_node_using_data_model(self, data_model_or_view_id, add_class_prefix)
537
- elif isinstance(data_model_or_view_id, ViewId):
538
- if not data_model_or_view_id.space:
539
- raise exceptions.SpaceNotDefined()
540
- if not data_model_or_view_id.external_id:
541
- raise exceptions.ViewExternalIdNotDefined()
542
- if not data_model_or_view_id.version:
543
- raise exceptions.ViewVersionNotDefined()
544
- return _to_node_using_view_id(self, data_model_or_view_id)
545
- else:
546
- space = self.model_json_schema().get("space", None)
547
- external_id = self.model_json_schema().get("external_id", None)
548
- version = self.model_json_schema().get("version", None)
549
-
550
- if not space:
551
- raise exceptions.SpaceNotDefined()
552
- if not external_id:
553
- raise exceptions.ViewExternalIdNotDefined()
554
- if not version:
555
- raise exceptions.ViewVersionNotDefined()
556
-
557
- return _to_node_using_view_id(self, ViewId(space, external_id, version))
558
-
559
-
560
- def _to_node_using_view_id(self, view_id: ViewId) -> NodeApply:
561
- attributes: dict = {
562
- attribute: (
563
- getattr(self, attribute).isoformat()
564
- if isinstance(getattr(self, attribute), date)
565
- else getattr(self, attribute)
566
- )
567
- for attribute in self.attributes
568
- }
569
-
570
- edges_one_to_one: dict = {}
571
-
572
- for edge in self.edges_one_to_one:
573
- if external_id := getattr(self, edge):
574
- edges_one_to_one[edge] = {
575
- "space": self.model_json_schema()["space"],
576
- "externalId": external_id,
577
- }
578
-
579
- return NodeApply(
580
- space=self.model_json_schema()["space"],
581
- external_id=self.external_id,
582
- sources=[
583
- NodeOrEdgeData(
584
- source=view_id,
585
- properties=attributes | edges_one_to_one,
586
- )
587
- ],
588
- )
589
-
590
-
591
- def _to_node_using_data_model(self, data_model, add_class_prefix) -> NodeApply:
592
- view_id: str = f"{self.model_json_schema()['space']}:{type(self).__name__}"
593
-
594
- if not set(self.attributes + self.edges_one_to_one).issubset(set(data_model.views[view_id].properties.keys())):
595
- raise exceptions.InstancePropertiesNotMatchingViewProperties(
596
- self.__class__.__name__,
597
- self.attributes + self.edges_one_to_one + self.edges_one_to_many,
598
- list(data_model.views[view_id].properties.keys()),
599
- )
600
-
601
- attributes: dict = {
602
- attribute: (
603
- getattr(self, attribute).isoformat()
604
- if isinstance(getattr(self, attribute), date)
605
- else getattr(self, attribute)
606
- )
607
- for attribute in self.attributes
608
- }
609
- if add_class_prefix:
610
- edges_one_to_one: dict = {}
611
- dm_view = data_model.views[view_id]
612
- if dm_view.properties:
613
- for edge in self.edges_one_to_one:
614
- mapped_property = dm_view.properties[edge]
615
- if isinstance(mapped_property, MappedPropertyApply):
616
- object_view = mapped_property.source
617
- if object_view:
618
- object_class_name = object_view.external_id
619
-
620
- if external_id := getattr(self, edge):
621
- edges_one_to_one[edge] = {
622
- "space": data_model.views[view_id].space,
623
- "externalId": add_class_prefix_to_xid(
624
- class_name=object_class_name,
625
- external_id=external_id,
626
- ),
627
- }
628
- else:
629
- edges_one_to_one = {}
630
-
631
- for edge in self.edges_one_to_one:
632
- if external_id := getattr(self, edge):
633
- edges_one_to_one[edge] = {
634
- "space": self.model_json_schema()["space"],
635
- "externalId": external_id,
636
- }
637
-
638
- return NodeApply(
639
- space=data_model.views[view_id].space,
640
- external_id=self.external_id,
641
- sources=[
642
- NodeOrEdgeData(
643
- source=data_model.views[view_id].as_id(),
644
- properties=attributes | edges_one_to_one,
645
- )
646
- ],
647
- )
648
-
649
-
650
- def to_edge(self, data_model: DMSSchemaComponents, add_class_prefix: bool = False) -> list[EdgeApply]:
651
- """Creates DMS edge from pydantic model."""
652
- edges: list[EdgeApply] = []
653
-
654
- def is_external_id_valid(external_id: str) -> bool:
655
- # should match "^[^\x00]{1,255}$" and not be None or none
656
- if external_id == "None" or external_id == "none":
657
- return False
658
- return bool(re.match(r"^[^\x00]{1,255}$", external_id))
659
-
660
- class_name: str = type(self).__name__
661
- view_id: str = f"{self.model_json_schema()['space']}:{class_name}"
662
-
663
- for edge_one_to_many in self.edges_one_to_many:
664
- edge_type_id = f"{class_name}.{edge_one_to_many}"
665
- if end_node_ids := getattr(self, edge_one_to_many):
666
- for end_node_id in end_node_ids:
667
- if not is_external_id_valid(end_node_id):
668
- warnings.warn(
669
- message=exceptions.EdgeConditionUnmet(edge_one_to_many).message,
670
- category=exceptions.EdgeConditionUnmet,
671
- stacklevel=2,
672
- )
673
- end_node_external_id = end_node_id
674
- if add_class_prefix:
675
- end_node_class_name = _get_end_node_class_name(data_model.views[view_id], edge_one_to_many)
676
- if end_node_class_name:
677
- end_node_external_id = add_class_prefix_to_xid(end_node_class_name, end_node_id)
678
- else:
679
- warnings.warn(
680
- message=exceptions.EdgeConditionUnmet(edge_one_to_many).message,
681
- category=exceptions.EdgeConditionUnmet,
682
- stacklevel=2,
683
- )
684
-
685
- external_id = f"{self.external_id}.{edge_one_to_many}.{end_node_external_id}"
686
- edge = EdgeApply(
687
- space=data_model.views[view_id].space,
688
- external_id=external_id if len(external_id) < 256 else create_sha256_hash(external_id),
689
- type=(data_model.views[view_id].space, edge_type_id),
690
- start_node=(data_model.views[view_id].space, self.external_id),
691
- end_node=(data_model.views[view_id].space, end_node_external_id),
692
- )
693
- edges.append(edge)
694
- return edges
695
-
696
-
697
- def _get_end_node_class_name(view: ViewApply, edge: str) -> str | None:
698
- """Get the class name of the end node of an edge."""
699
- if view.properties is None:
700
- return None
701
- mapped_instance = view.properties[edge]
702
- if isinstance(mapped_instance, SingleHopConnectionDefinitionApply) and mapped_instance.source:
703
- return mapped_instance.source.external_id
704
- return None
705
-
706
-
707
- @classmethod # type: ignore
708
- @property
709
- def model_name(cls) -> str | None:
710
- """Returns the name of the model if one exists"""
711
- return cls.model_json_schema().get("class_name", None)
712
-
713
-
714
- @classmethod # type: ignore
715
- @property
716
- def model_external_id(cls) -> str | None:
717
- """Returns the external id of the model if one exists"""
718
- return cls.model_json_schema().get("class_id", None)
719
-
720
-
721
- @classmethod # type: ignore
722
- @property
723
- def model_description(cls) -> str | None:
724
- """Returns the description of the model if one exists"""
725
- return cls.model_json_schema().get("description", None)
726
-
727
-
728
- @classmethod # type: ignore
729
- def get_field_description(cls, field_id: str) -> str | None:
730
- """Returns description of the field if one exists"""
731
- if field_id in cls.model_fields:
732
- return cls.model_fields[field_id].description
733
- else:
734
- return None
735
-
736
-
737
- @classmethod # type: ignore
738
- def get_field_name(cls, field_id: str) -> str | None:
739
- """Returns name of the field if one exists"""
740
- if field_id in cls.model_fields and cls.model_fields[field_id].json_schema_extra:
741
- if "property_name" in cls.model_fields[field_id].json_schema_extra:
742
- return cls.model_fields[field_id].json_schema_extra["property_name"]
743
- return None
744
-
745
-
746
- def add_class_prefix_to_xid(class_name: str, external_id: str) -> str:
747
- """Adds class name as prefix to the external_id"""
748
- return f"{class_name}_{external_id}"