cognite-neat 0.121.0__py3-none-any.whl → 0.121.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cognite-neat might be problematic. Click here for more details.

Files changed (150) hide show
  1. cognite/neat/_version.py +1 -1
  2. cognite/neat/core/_client/_api/statistics.py +91 -0
  3. cognite/neat/core/_client/_api_client.py +2 -0
  4. cognite/neat/core/_client/data_classes/statistics.py +125 -0
  5. cognite/neat/core/_client/testing.py +4 -0
  6. cognite/neat/core/_constants.py +6 -7
  7. cognite/neat/core/{_rules → _data_model}/_constants.py +25 -18
  8. cognite/neat/core/_data_model/_shared.py +59 -0
  9. cognite/neat/core/_data_model/analysis/__init__.py +3 -0
  10. cognite/neat/core/{_rules → _data_model}/analysis/_base.py +202 -195
  11. cognite/neat/core/{_rules → _data_model}/catalog/__init__.py +1 -1
  12. cognite/neat/core/{_rules → _data_model}/exporters/__init__.py +5 -5
  13. cognite/neat/core/{_rules → _data_model}/exporters/_base.py +10 -8
  14. cognite/neat/core/{_rules/exporters/_rules2dms.py → _data_model/exporters/_data_model2dms.py} +22 -18
  15. cognite/neat/core/{_rules/exporters/_rules2excel.py → _data_model/exporters/_data_model2excel.py} +61 -56
  16. cognite/neat/core/{_rules/exporters/_rules2instance_template.py → _data_model/exporters/_data_model2instance_template.py} +11 -9
  17. cognite/neat/core/{_rules/exporters/_rules2ontology.py → _data_model/exporters/_data_model2ontology.py} +64 -61
  18. cognite/neat/core/{_rules/exporters/_rules2yaml.py → _data_model/exporters/_data_model2yaml.py} +21 -18
  19. cognite/neat/core/{_rules → _data_model}/importers/__init__.py +6 -8
  20. cognite/neat/core/{_rules → _data_model}/importers/_base.py +8 -6
  21. cognite/neat/core/_data_model/importers/_base_file_reader.py +56 -0
  22. cognite/neat/core/{_rules/importers/_yaml2rules.py → _data_model/importers/_dict2data_model.py} +41 -21
  23. cognite/neat/core/{_rules/importers/_dms2rules.py → _data_model/importers/_dms2data_model.py} +79 -66
  24. cognite/neat/core/{_rules/importers/_dtdl2rules → _data_model/importers/_dtdl2data_model}/dtdl_converter.py +41 -41
  25. cognite/neat/core/{_rules/importers/_dtdl2rules → _data_model/importers/_dtdl2data_model}/dtdl_importer.py +16 -16
  26. cognite/neat/core/{_rules/importers/_dtdl2rules → _data_model/importers/_dtdl2data_model}/spec.py +3 -3
  27. cognite/neat/core/{_rules → _data_model}/importers/_rdf/_base.py +18 -16
  28. cognite/neat/core/{_rules → _data_model}/importers/_rdf/_imf2rules.py +17 -17
  29. cognite/neat/core/{_rules → _data_model}/importers/_rdf/_inference2rules.py +50 -50
  30. cognite/neat/core/{_rules → _data_model}/importers/_rdf/_owl2rules.py +14 -14
  31. cognite/neat/core/{_rules → _data_model}/importers/_rdf/_shared.py +25 -25
  32. cognite/neat/core/{_rules/importers/_spreadsheet2rules.py → _data_model/importers/_spreadsheet2data_model.py} +69 -38
  33. cognite/neat/core/_data_model/models/__init__.py +36 -0
  34. cognite/neat/core/{_rules/models/_base_input.py → _data_model/models/_base_unverified.py} +12 -12
  35. cognite/neat/core/{_rules/models/_base_rules.py → _data_model/models/_base_verified.py} +13 -13
  36. cognite/neat/core/{_rules → _data_model}/models/_types.py +13 -13
  37. cognite/neat/core/_data_model/models/conceptual/__init__.py +25 -0
  38. cognite/neat/core/{_rules/models/information/_rules_input.py → _data_model/models/conceptual/_unverified.py} +46 -43
  39. cognite/neat/core/{_rules/models/information → _data_model/models/conceptual}/_validation.py +93 -79
  40. cognite/neat/core/{_rules/models/information/_rules.py → _data_model/models/conceptual/_verified.py} +83 -83
  41. cognite/neat/core/{_rules → _data_model}/models/data_types.py +4 -4
  42. cognite/neat/core/{_rules → _data_model}/models/entities/__init__.py +8 -8
  43. cognite/neat/core/{_rules → _data_model}/models/entities/_loaders.py +12 -11
  44. cognite/neat/core/{_rules → _data_model}/models/entities/_multi_value.py +7 -7
  45. cognite/neat/core/{_rules → _data_model}/models/entities/_single_value.py +45 -39
  46. cognite/neat/core/{_rules → _data_model}/models/entities/_types.py +9 -3
  47. cognite/neat/core/{_rules → _data_model}/models/entities/_wrapped.py +3 -3
  48. cognite/neat/core/{_rules → _data_model}/models/mapping/_classic2core.py +12 -9
  49. cognite/neat/core/_data_model/models/physical/__init__.py +40 -0
  50. cognite/neat/core/{_rules/models/dms → _data_model/models/physical}/_exporter.py +83 -64
  51. cognite/neat/core/{_rules/models/dms/_rules_input.py → _data_model/models/physical/_unverified.py} +56 -44
  52. cognite/neat/core/{_rules/models/dms → _data_model/models/physical}/_validation.py +20 -17
  53. cognite/neat/core/{_rules/models/dms/_rules.py → _data_model/models/physical/_verified.py} +79 -71
  54. cognite/neat/core/{_rules → _data_model}/transformers/__init__.py +27 -23
  55. cognite/neat/core/{_rules → _data_model}/transformers/_base.py +29 -19
  56. cognite/neat/core/{_rules → _data_model}/transformers/_converters.py +758 -659
  57. cognite/neat/core/{_rules → _data_model}/transformers/_mapping.py +79 -60
  58. cognite/neat/core/_data_model/transformers/_verification.py +120 -0
  59. cognite/neat/core/{_graph → _instances}/extractors/_base.py +2 -2
  60. cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_base.py +1 -1
  61. cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_classic.py +17 -11
  62. cognite/neat/core/{_graph → _instances}/extractors/_dms_graph.py +47 -39
  63. cognite/neat/core/{_graph → _instances}/extractors/_mock_graph_generator.py +102 -99
  64. cognite/neat/core/{_graph → _instances}/extractors/_rdf_file.py +2 -2
  65. cognite/neat/core/{_graph → _instances}/loaders/_base.py +2 -2
  66. cognite/neat/core/{_graph → _instances}/loaders/_rdf2dms.py +16 -14
  67. cognite/neat/core/{_graph → _instances}/transformers/_base.py +7 -4
  68. cognite/neat/core/{_graph → _instances}/transformers/_classic_cdf.py +1 -1
  69. cognite/neat/core/{_graph → _instances}/transformers/_value_type.py +2 -6
  70. cognite/neat/core/_issues/_base.py +4 -4
  71. cognite/neat/core/_issues/errors/__init__.py +2 -2
  72. cognite/neat/core/_issues/errors/_wrapper.py +2 -2
  73. cognite/neat/core/_issues/warnings/__init__.py +2 -0
  74. cognite/neat/core/_issues/warnings/_models.py +4 -4
  75. cognite/neat/core/_issues/warnings/_properties.py +7 -0
  76. cognite/neat/core/_store/__init__.py +3 -3
  77. cognite/neat/core/_store/{_rules_store.py → _data_model.py} +128 -121
  78. cognite/neat/core/_store/{_graph_store.py → _instance.py} +7 -8
  79. cognite/neat/core/_store/_provenance.py +2 -2
  80. cognite/neat/core/_store/exceptions.py +4 -4
  81. cognite/neat/core/_utils/rdf_.py +14 -0
  82. cognite/neat/core/_utils/spreadsheet.py +1 -1
  83. cognite/neat/core/_utils/text.py +2 -2
  84. cognite/neat/session/_base.py +29 -25
  85. cognite/neat/session/_drop.py +3 -3
  86. cognite/neat/session/_fix.py +2 -2
  87. cognite/neat/session/_inspect.py +5 -5
  88. cognite/neat/session/_mapping.py +11 -9
  89. cognite/neat/session/_prepare.py +4 -4
  90. cognite/neat/session/_read.py +15 -15
  91. cognite/neat/session/_set.py +5 -5
  92. cognite/neat/session/_show.py +11 -11
  93. cognite/neat/session/_state.py +17 -17
  94. cognite/neat/session/_subset.py +14 -11
  95. cognite/neat/session/_template.py +19 -19
  96. cognite/neat/session/_to.py +21 -21
  97. cognite/neat/session/_wizard.py +1 -1
  98. {cognite_neat-0.121.0.dist-info → cognite_neat-0.121.2.dist-info}/METADATA +1 -1
  99. cognite_neat-0.121.2.dist-info/RECORD +189 -0
  100. cognite/neat/core/_rules/_shared.py +0 -43
  101. cognite/neat/core/_rules/analysis/__init__.py +0 -3
  102. cognite/neat/core/_rules/exporters/_validation.py +0 -14
  103. cognite/neat/core/_rules/models/__init__.py +0 -34
  104. cognite/neat/core/_rules/models/dms/__init__.py +0 -32
  105. cognite/neat/core/_rules/models/information/__init__.py +0 -20
  106. cognite/neat/core/_rules/transformers/_verification.py +0 -111
  107. cognite_neat-0.121.0.dist-info/RECORD +0 -187
  108. /cognite/neat/core/{_graph → _data_model}/__init__.py +0 -0
  109. /cognite/neat/core/{_rules → _data_model}/catalog/classic_model.xlsx +0 -0
  110. /cognite/neat/core/{_rules/catalog/info-rules-imf.xlsx → _data_model/catalog/conceptual-imf-data-model.xlsx} +0 -0
  111. /cognite/neat/core/{_rules → _data_model}/catalog/hello_world_pump.xlsx +0 -0
  112. /cognite/neat/core/{_rules/importers/_dtdl2rules → _data_model/importers/_dtdl2data_model}/__init__.py +0 -0
  113. /cognite/neat/core/{_rules/importers/_dtdl2rules → _data_model/importers/_dtdl2data_model}/_unit_lookup.py +0 -0
  114. /cognite/neat/core/{_rules → _data_model}/importers/_rdf/__init__.py +0 -0
  115. /cognite/neat/core/{_rules → _data_model}/models/entities/_constants.py +0 -0
  116. /cognite/neat/core/{_rules → _data_model}/models/mapping/__init__.py +0 -0
  117. /cognite/neat/core/{_rules → _data_model}/models/mapping/_classic2core.yaml +0 -0
  118. /cognite/neat/core/{_graph/extractors/_classic_cdf → _instances}/__init__.py +0 -0
  119. /cognite/neat/core/{_graph → _instances}/_shared.py +0 -0
  120. /cognite/neat/core/{_graph → _instances}/_tracking/__init__.py +0 -0
  121. /cognite/neat/core/{_graph → _instances}/_tracking/base.py +0 -0
  122. /cognite/neat/core/{_graph → _instances}/_tracking/log.py +0 -0
  123. /cognite/neat/core/{_graph → _instances}/examples/Knowledge-Graph-Nordic44-dirty.xml +0 -0
  124. /cognite/neat/core/{_graph → _instances}/examples/Knowledge-Graph-Nordic44.xml +0 -0
  125. /cognite/neat/core/{_graph → _instances}/examples/__init__.py +0 -0
  126. /cognite/neat/core/{_graph → _instances}/examples/skos-capturing-sheet-wind-topics.xlsx +0 -0
  127. /cognite/neat/core/{_graph → _instances}/extractors/__init__.py +0 -0
  128. /cognite/neat/core/{_rules → _instances/extractors/_classic_cdf}/__init__.py +0 -0
  129. /cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_assets.py +0 -0
  130. /cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_data_sets.py +0 -0
  131. /cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_events.py +0 -0
  132. /cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_files.py +0 -0
  133. /cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_labels.py +0 -0
  134. /cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_relationships.py +0 -0
  135. /cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_sequences.py +0 -0
  136. /cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_timeseries.py +0 -0
  137. /cognite/neat/core/{_graph → _instances}/extractors/_dict.py +0 -0
  138. /cognite/neat/core/{_graph → _instances}/extractors/_dms.py +0 -0
  139. /cognite/neat/core/{_graph → _instances}/extractors/_raw.py +0 -0
  140. /cognite/neat/core/{_graph → _instances}/loaders/__init__.py +0 -0
  141. /cognite/neat/core/{_graph → _instances}/queries/__init__.py +0 -0
  142. /cognite/neat/core/{_graph → _instances}/queries/_base.py +0 -0
  143. /cognite/neat/core/{_graph → _instances}/queries/_queries.py +0 -0
  144. /cognite/neat/core/{_graph → _instances}/queries/_select.py +0 -0
  145. /cognite/neat/core/{_graph → _instances}/queries/_update.py +0 -0
  146. /cognite/neat/core/{_graph → _instances}/transformers/__init__.py +0 -0
  147. /cognite/neat/core/{_graph → _instances}/transformers/_prune_graph.py +0 -0
  148. /cognite/neat/core/{_graph → _instances}/transformers/_rdfpath.py +0 -0
  149. {cognite_neat-0.121.0.dist-info → cognite_neat-0.121.2.dist-info}/WHEEL +0 -0
  150. {cognite_neat-0.121.0.dist-info → cognite_neat-0.121.2.dist-info}/licenses/LICENSE +0 -0
@@ -1,18 +1,30 @@
1
1
  """This module performs importing of graph to TransformationRules pydantic class.
2
2
  In more details, it traverses the graph and abstracts class and properties, basically
3
- generating a list of rules based on which nodes that form the graph are made.
3
+ generating a list of data_model based on which nodes that form the graph are made.
4
4
  """
5
5
 
6
+ import tempfile
6
7
  from collections import UserDict, defaultdict
7
8
  from dataclasses import dataclass
8
9
  from pathlib import Path
9
10
  from typing import Literal, cast
10
11
 
11
12
  import pandas as pd
12
- from cognite.client.utils._importing import local_import
13
+ from openpyxl import load_workbook
14
+ from openpyxl.worksheet.worksheet import Worksheet
13
15
  from pandas import ExcelFile
14
16
  from rdflib import Namespace, URIRef
15
17
 
18
+ from cognite.neat.core._data_model._shared import (
19
+ ImportedDataModel,
20
+ T_UnverifiedDataModel,
21
+ )
22
+ from cognite.neat.core._data_model.models import (
23
+ INPUT_RULES_BY_ROLE,
24
+ VERIFIED_RULES_BY_ROLE,
25
+ RoleTypes,
26
+ SchemaCompleteness,
27
+ )
16
28
  from cognite.neat.core._issues import IssueList, MultiValueError
17
29
  from cognite.neat.core._issues.errors import (
18
30
  FileMissingRequiredFieldError,
@@ -20,13 +32,6 @@ from cognite.neat.core._issues.errors import (
20
32
  FileReadError,
21
33
  )
22
34
  from cognite.neat.core._issues.warnings import FileMissingRequiredFieldWarning
23
- from cognite.neat.core._rules._shared import ReadRules, T_InputRules
24
- from cognite.neat.core._rules.models import (
25
- INPUT_RULES_BY_ROLE,
26
- VERIFIED_RULES_BY_ROLE,
27
- RoleTypes,
28
- SchemaCompleteness,
29
- )
30
35
  from cognite.neat.core._utils.spreadsheet import SpreadsheetRead, read_individual_sheet
31
36
  from cognite.neat.core._utils.text import humanize_collection
32
37
 
@@ -37,11 +42,11 @@ SOURCE_SHEET__TARGET_FIELD__HEADERS = [
37
42
  "Properties",
38
43
  "Properties",
39
44
  {
40
- RoleTypes.information: ["Class", "Property"],
45
+ RoleTypes.information: ["Concept", "Property"],
41
46
  RoleTypes.dms: ["View", "View Property"],
42
47
  },
43
48
  ),
44
- ("Classes", "Classes", ["Class"]),
49
+ ("Concepts", "Concepts", ["Concept"]),
45
50
  ("Containers", "Containers", ["Container"]),
46
51
  ("Views", "Views", ["View"]),
47
52
  ("Enum", "Enum", ["Collection"]),
@@ -243,8 +248,8 @@ class SpreadsheetReader:
243
248
  return sheets, read_info_by_sheet
244
249
 
245
250
 
246
- class ExcelImporter(BaseImporter[T_InputRules]):
247
- """Import rules from an Excel file.
251
+ class ExcelImporter(BaseImporter[T_UnverifiedDataModel]):
252
+ """Import data_model from an Excel file.
248
253
 
249
254
  Args:
250
255
  filepath (Path): The path to the Excel file.
@@ -253,14 +258,15 @@ class ExcelImporter(BaseImporter[T_InputRules]):
253
258
  def __init__(self, filepath: Path):
254
259
  self.filepath = filepath
255
260
 
256
- def to_rules(self) -> ReadRules[T_InputRules]:
261
+ def to_data_model(self) -> ImportedDataModel[T_UnverifiedDataModel]:
257
262
  issue_list = IssueList(title=f"'{self.filepath.name}'")
258
263
  if not self.filepath.exists():
259
264
  raise FileNotFoundNeatError(self.filepath)
260
265
 
266
+ self.filepath = self._make_forward_compatible_spreadsheet(self.filepath)
267
+
261
268
  with pd.ExcelFile(self.filepath) as excel_file:
262
269
  user_reader = SpreadsheetReader(issue_list)
263
-
264
270
  user_read = user_reader.read(excel_file, self.filepath)
265
271
 
266
272
  issue_list.trigger_warnings()
@@ -268,15 +274,23 @@ class ExcelImporter(BaseImporter[T_InputRules]):
268
274
  raise MultiValueError(issue_list.errors)
269
275
 
270
276
  if user_read is None:
271
- return ReadRules(None, {})
277
+ return ImportedDataModel(None, {})
272
278
 
273
279
  sheets = user_read.sheets
274
280
  original_role = user_read.role
275
281
  read_info_by_sheet = user_read.read_info_by_sheet
276
282
 
277
- rules_cls = INPUT_RULES_BY_ROLE[original_role]
278
- rules = cast(T_InputRules, rules_cls.load(sheets))
279
- return ReadRules(rules, read_info_by_sheet)
283
+ data_model_cls = INPUT_RULES_BY_ROLE[original_role]
284
+ data_model = cast(T_UnverifiedDataModel, data_model_cls.load(sheets))
285
+
286
+ # Delete the temporary file if it was created
287
+ if "temp_neat_file" in self.filepath.name:
288
+ try:
289
+ self.filepath.unlink()
290
+ except Exception as e:
291
+ issue_list.append(FileReadError(self.filepath, f"Failed to delete temporary file: {e}"))
292
+
293
+ return ImportedDataModel(data_model, read_info_by_sheet)
280
294
 
281
295
  @property
282
296
  def description(self) -> str:
@@ -286,30 +300,47 @@ class ExcelImporter(BaseImporter[T_InputRules]):
286
300
  def source_uri(self) -> URIRef:
287
301
  return URIRef(f"file://{self.filepath.name}")
288
302
 
303
+ def _make_forward_compatible_spreadsheet(self, filepath: Path) -> Path:
304
+ """Makes the spreadsheet forward compatible by renaming legacy class with concept
289
305
 
290
- class GoogleSheetImporter(BaseImporter[T_InputRules]):
291
- """Import rules from a Google Sheet.
306
+ Args:
307
+ filepath (Path): The path to the Excel file.
292
308
 
293
- .. warning::
309
+ """
294
310
 
295
- This importer is experimental and may not work as expected.
311
+ workbook = load_workbook(filepath)
296
312
 
297
- Args:
298
- sheet_id (str): The Google Sheet ID.
299
- skiprows (int): The number of rows to skip when reading the Google Sheet.
300
- """
313
+ if "Classes" in workbook.sheetnames:
314
+ print(
315
+ (
316
+ "You are using a legacy spreadsheet format, "
317
+ "which we will support until v1.0 of neat."
318
+ " Please update your spreadsheet to the new format."
319
+ ),
320
+ )
321
+ _replace_class_with_concept_cell(workbook["Classes"])
322
+ sheet = workbook["Classes"]
323
+ sheet.title = "Concepts"
301
324
 
302
- def __init__(self, sheet_id: str, skiprows: int = 1):
303
- self.sheet_id = sheet_id
304
- self.skiprows = skiprows
325
+ if "Properties" in workbook.sheetnames:
326
+ _replace_class_with_concept_cell(workbook["Properties"])
305
327
 
306
- def to_rules(self) -> ReadRules[T_InputRules]:
307
- raise NotImplementedError("Google Sheet Importer is not yet implemented.")
328
+ with tempfile.NamedTemporaryFile(prefix="temp_neat_file", suffix=".xlsx", delete=False) as temp_file:
329
+ workbook.save(temp_file.name)
330
+ return Path(temp_file.name)
308
331
 
309
- def _get_sheets(self) -> dict[str, pd.DataFrame]:
310
- local_import("gspread", "google")
311
- import gspread # type: ignore[import]
332
+ else:
333
+ return filepath
334
+
335
+
336
+ def _replace_class_with_concept_cell(sheet: Worksheet) -> None:
337
+ """
338
+ Replaces the word "Class" with "Concept" in the first row of the given sheet.
312
339
 
313
- client_google = gspread.service_account()
314
- google_sheet = client_google.open_by_key(self.sheet_id)
315
- return {worksheet.title: pd.DataFrame(worksheet.get_all_records()) for worksheet in google_sheet.worksheets()}
340
+ Args:
341
+ sheet (Worksheet): The sheet in which to replace the word "Class".
342
+ """
343
+ for row in sheet.iter_rows():
344
+ for cell in row:
345
+ if cell.value == "Class":
346
+ cell.value = "Concept"
@@ -0,0 +1,36 @@
1
+ from cognite.neat.core._client.data_classes.schema import DMSSchema
2
+ from cognite.neat.core._data_model.models.conceptual._unverified import (
3
+ UnverifiedConceptualDataModel,
4
+ )
5
+ from cognite.neat.core._data_model.models.conceptual._verified import (
6
+ ConceptualDataModel,
7
+ )
8
+
9
+ from ._base_verified import DataModelType, ExtensionCategory, RoleTypes, SchemaCompleteness, SheetList, SheetRow
10
+ from .physical._unverified import UnverifiedPhysicalDataModel
11
+ from .physical._verified import PhysicalDataModel
12
+
13
+ INPUT_RULES_BY_ROLE: dict[RoleTypes, type[UnverifiedConceptualDataModel] | type[UnverifiedPhysicalDataModel]] = {
14
+ RoleTypes.information: UnverifiedConceptualDataModel,
15
+ RoleTypes.dms: UnverifiedPhysicalDataModel,
16
+ }
17
+ VERIFIED_RULES_BY_ROLE: dict[RoleTypes, type[ConceptualDataModel] | type[PhysicalDataModel]] = {
18
+ RoleTypes.information: ConceptualDataModel,
19
+ RoleTypes.dms: PhysicalDataModel,
20
+ }
21
+
22
+
23
+ __all__ = [
24
+ "INPUT_RULES_BY_ROLE",
25
+ "ConceptualDataModel",
26
+ "DMSSchema",
27
+ "DataModelType",
28
+ "ExtensionCategory",
29
+ "PhysicalDataModel",
30
+ "RoleTypes",
31
+ "SchemaCompleteness",
32
+ "SheetList",
33
+ "SheetRow",
34
+ "UnverifiedConceptualDataModel",
35
+ "UnverifiedPhysicalDataModel",
36
+ ]
@@ -1,15 +1,15 @@
1
- """Module for base classes for the input models.
1
+ """Module for base classes for the unverified models.
2
2
 
3
- The philosophy of the input models is:
3
+ The philosophy of the unverified data models is:
4
4
 
5
- * Provide an easy way to input rules. The type hints are made to be human-friendly, for example, Literal instead of
6
- Enum.
5
+ * Provide an easy way to read data model into neat. The type hints are made to be human-friendly,
6
+ for example, Literal instead of Enum.
7
7
  * The .dump() method should fill out defaults and have shortcuts. For example, if the prefix is not provided for
8
8
  a class, then the prefix from the metadata is used. For views, if the class is not provided, it is assumed to
9
9
  be the same as the view.
10
10
 
11
- The base classes are to make it easy to create the input models with default behavior. They are also used for
12
- testing to ensure that input models correctly map to the verified rules models.
11
+ The base classes are to make it easy to create the unverified data models with default behavior.
12
+ They are also used for testing to ensure that unverified models correctly map to the verified models.
13
13
  """
14
14
 
15
15
  import sys
@@ -20,19 +20,19 @@ from typing import Any, Generic, TypeVar, Union, cast, get_args, get_origin, ove
20
20
 
21
21
  import pandas as pd
22
22
 
23
- from ._base_rules import BaseRules, SchemaModel
23
+ from ._base_verified import BaseVerifiedDataModel, SchemaModel
24
24
 
25
25
  if sys.version_info >= (3, 11):
26
26
  from typing import Self
27
27
  else:
28
28
  from typing_extensions import Self
29
29
 
30
- T_BaseRules = TypeVar("T_BaseRules", bound=BaseRules)
30
+ T_BaseRules = TypeVar("T_BaseRules", bound=BaseVerifiedDataModel)
31
31
  T_RuleModel = TypeVar("T_RuleModel", bound=SchemaModel)
32
32
 
33
33
 
34
34
  @dataclass
35
- class InputRules(Generic[T_BaseRules], ABC):
35
+ class UnverifiedDataModel(Generic[T_BaseRules], ABC):
36
36
  """Input rules are raw data that is not yet validated."""
37
37
 
38
38
  @classmethod
@@ -110,7 +110,7 @@ class InputRules(Generic[T_BaseRules], ABC):
110
110
  def _dataclass_fields(self) -> list[Field]:
111
111
  return list(fields(self))
112
112
 
113
- def as_verified_rules(self) -> T_BaseRules:
113
+ def as_verified_data_model(self) -> T_BaseRules:
114
114
  cls_ = self._get_verified_cls()
115
115
  return cls_.model_validate(self.dump())
116
116
 
@@ -127,11 +127,11 @@ class InputRules(Generic[T_BaseRules], ABC):
127
127
  return output
128
128
 
129
129
 
130
- T_InputRules = TypeVar("T_InputRules", bound=InputRules)
130
+ T_InputRules = TypeVar("T_InputRules", bound=UnverifiedDataModel)
131
131
 
132
132
 
133
133
  @dataclass
134
- class InputComponent(ABC, Generic[T_RuleModel]):
134
+ class UnverifiedComponent(ABC, Generic[T_RuleModel]):
135
135
  @classmethod
136
136
  @abstractmethod
137
137
  def _get_verified_cls(cls) -> type[T_RuleModel]:
@@ -27,18 +27,18 @@ from pydantic_core import core_schema
27
27
  from rdflib import Namespace, URIRef
28
28
 
29
29
  from cognite.neat.core._constants import DEFAULT_NAMESPACE
30
- from cognite.neat.core._rules.models._types import (
30
+ from cognite.neat.core._data_model.models._types import (
31
31
  ContainerEntityType,
32
32
  DataModelExternalIdType,
33
- DmsPropertyType,
33
+ PhysicalPropertyType,
34
34
  SpaceType,
35
35
  StrListType,
36
36
  URIRefType,
37
37
  VersionType,
38
38
  ViewEntityType,
39
39
  )
40
- from cognite.neat.core._rules.models.data_types import DataType
41
- from cognite.neat.core._rules.models.entities import (
40
+ from cognite.neat.core._data_model.models.data_types import DataType
41
+ from cognite.neat.core._data_model.models.entities import (
42
42
  EdgeEntity,
43
43
  ReverseConnectionEntity,
44
44
  ViewEntity,
@@ -99,7 +99,7 @@ class DataModelType(StrEnum):
99
99
  enterprise = "enterprise"
100
100
 
101
101
 
102
- class DataModelAspect(StrEnum):
102
+ class DataModelLevel(StrEnum):
103
103
  conceptual = "conceptual"
104
104
  logical = "logical"
105
105
  physical = "physical"
@@ -134,13 +134,13 @@ class SchemaModel(BaseModel):
134
134
  return value
135
135
 
136
136
 
137
- class BaseMetadata(SchemaModel):
137
+ class BaseVerifiedMetadata(SchemaModel):
138
138
  """
139
139
  Metadata model for data model
140
140
  """
141
141
 
142
142
  role: ClassVar[RoleTypes] = Field(description="Role of the person creating the data model")
143
- aspect: ClassVar[DataModelAspect] = Field(description="Aspect of the data model")
143
+ level: ClassVar[DataModelLevel] = Field(description="Aspect of the data model")
144
144
  space: SpaceType = Field(description="The space where the data model is defined")
145
145
  external_id: DataModelExternalIdType = Field(
146
146
  alias="externalId", description="External identifier for the data model"
@@ -223,7 +223,7 @@ class BaseMetadata(SchemaModel):
223
223
  Unlike namespace, the identifier does not end with "/" or "#".
224
224
 
225
225
  """
226
- return DEFAULT_NAMESPACE[f"data-model/verified/{self.aspect}/{self.space}/{self.external_id}/{self.version}"]
226
+ return DEFAULT_NAMESPACE[f"data-model/verified/{self.level}/{self.space}/{self.external_id}/{self.version}"]
227
227
 
228
228
  @property
229
229
  def namespace(self) -> Namespace:
@@ -237,7 +237,7 @@ class BaseMetadata(SchemaModel):
237
237
  return repr(self.as_data_model_id())
238
238
 
239
239
  @classmethod
240
- def default(cls) -> "BaseMetadata":
240
+ def default(cls) -> "BaseVerifiedMetadata":
241
241
  """Returns a default instance of the metadata model."""
242
242
  now = datetime.now()
243
243
  return cls(
@@ -252,7 +252,7 @@ class BaseMetadata(SchemaModel):
252
252
  )
253
253
 
254
254
 
255
- class BaseRules(SchemaModel, ABC):
255
+ class BaseVerifiedDataModel(SchemaModel, ABC):
256
256
  """
257
257
  Rules is a core concept in `neat`. This represents fusion of data model
258
258
  definitions and (optionally) the transformation rules used to transform the data/graph
@@ -265,7 +265,7 @@ class BaseRules(SchemaModel, ABC):
265
265
  metadata: Data model metadata
266
266
  """
267
267
 
268
- metadata: BaseMetadata
268
+ metadata: BaseVerifiedMetadata
269
269
 
270
270
  @classmethod
271
271
  def headers_by_sheet(cls, by_alias: bool = False) -> dict[str, list[str]]:
@@ -438,7 +438,7 @@ ExtensionCategoryType = Annotated[
438
438
  # Immutable such that this can be used as a key in a dictionary
439
439
  class ContainerProperty(BaseModel, frozen=True):
440
440
  container: ContainerEntityType
441
- property_: DmsPropertyType
441
+ property_: PhysicalPropertyType
442
442
 
443
443
 
444
444
  class ContainerDestinationProperty(ContainerProperty, frozen=True):
@@ -451,4 +451,4 @@ class ViewRef(BaseModel, frozen=True):
451
451
 
452
452
 
453
453
  class ViewProperty(ViewRef, frozen=True):
454
- property_: DmsPropertyType
454
+ property_: PhysicalPropertyType
@@ -14,23 +14,23 @@ from pydantic import (
14
14
  )
15
15
  from pydantic.functional_serializers import PlainSerializer
16
16
 
17
- from cognite.neat.core._issues.errors import RegexViolationError
18
- from cognite.neat.core._issues.warnings import RegexViolationWarning
19
- from cognite.neat.core._rules._constants import (
17
+ from cognite.neat.core._data_model._constants import (
20
18
  DATA_MODEL_COMPLIANCE_REGEX,
21
19
  PATTERNS,
22
20
  PREFIX_COMPLIANCE_REGEX,
23
21
  VERSION_COMPLIANCE_REGEX,
24
22
  EntityTypes,
25
23
  )
26
- from cognite.neat.core._rules.models.entities._multi_value import MultiValueTypeInfo
27
- from cognite.neat.core._rules.models.entities._single_value import (
28
- ClassEntity,
24
+ from cognite.neat.core._data_model.models.entities._multi_value import MultiValueTypeInfo
25
+ from cognite.neat.core._data_model.models.entities._single_value import (
26
+ ConceptEntity,
29
27
  ContainerEntity,
30
28
  ViewEntity,
31
29
  )
30
+ from cognite.neat.core._issues.errors import RegexViolationError
31
+ from cognite.neat.core._issues.warnings import RegexViolationWarning
32
32
 
33
- Entities: TypeAlias = ClassEntity | ViewEntity | ContainerEntity
33
+ Entities: TypeAlias = ConceptEntity | ViewEntity | ContainerEntity
34
34
  T_Entities = TypeVar("T_Entities", bound=Entities)
35
35
 
36
36
 
@@ -135,13 +135,13 @@ SpaceType = Annotated[
135
135
  AfterValidator(_external_id_validation_factory(EntityTypes.space, "")),
136
136
  ]
137
137
 
138
- InformationPropertyType = Annotated[
138
+ ConceptualPropertyType = Annotated[
139
139
  str,
140
- AfterValidator(_external_id_validation_factory(EntityTypes.information_property, "Property column in properties")),
140
+ AfterValidator(_external_id_validation_factory(EntityTypes.conceptual_property, "Property column in properties")),
141
141
  ]
142
- DmsPropertyType = Annotated[
142
+ PhysicalPropertyType = Annotated[
143
143
  str,
144
- AfterValidator(_external_id_validation_factory(EntityTypes.dms_property, "Property column in properties")),
144
+ AfterValidator(_external_id_validation_factory(EntityTypes.physical_property, "Property column in properties")),
145
145
  ]
146
146
 
147
147
 
@@ -152,7 +152,7 @@ def _entity_validation(value: Entities, location: str) -> Entities:
152
152
  return value
153
153
 
154
154
 
155
- ClassEntityType = Annotated[ClassEntity, AfterValidator(lambda v: _entity_validation(v, "the Class column"))]
155
+ ConceptEntityType = Annotated[ConceptEntity, AfterValidator(lambda v: _entity_validation(v, "the Class column"))]
156
156
  ViewEntityType = Annotated[ViewEntity, AfterValidator(lambda v: _entity_validation(v, "the View column"))]
157
157
  ContainerEntityType = Annotated[
158
158
  ContainerEntity, AfterValidator(lambda v: _entity_validation(v, "the Container column"))
@@ -161,7 +161,7 @@ ContainerEntityType = Annotated[
161
161
 
162
162
  def _multi_value_type_validation(value: MultiValueTypeInfo, location: str) -> MultiValueTypeInfo:
163
163
  for type_ in value.types:
164
- if isinstance(type_, ClassEntity):
164
+ if isinstance(type_, ConceptEntity):
165
165
  _entity_validation(type_, location)
166
166
  return value
167
167
 
@@ -0,0 +1,25 @@
1
+ from ._unverified import (
2
+ UnverifiedConcept,
3
+ UnverifiedConceptualDataModel,
4
+ UnverifiedConceptualMetadata,
5
+ UnverifiedConceptualProperty,
6
+ )
7
+ from ._validation import ConceptualValidation
8
+ from ._verified import (
9
+ Concept,
10
+ ConceptualDataModel,
11
+ ConceptualMetadata,
12
+ ConceptualProperty,
13
+ )
14
+
15
+ __all__ = [
16
+ "Concept",
17
+ "ConceptualDataModel",
18
+ "ConceptualMetadata",
19
+ "ConceptualProperty",
20
+ "ConceptualValidation",
21
+ "UnverifiedConcept",
22
+ "UnverifiedConceptualDataModel",
23
+ "UnverifiedConceptualMetadata",
24
+ "UnverifiedConceptualProperty",
25
+ ]
@@ -7,26 +7,29 @@ from cognite.client import data_modeling as dm
7
7
  from rdflib import Namespace, URIRef
8
8
 
9
9
  from cognite.neat.core._constants import DEFAULT_NAMESPACE
10
- from cognite.neat.core._rules.models._base_input import InputComponent, InputRules
11
- from cognite.neat.core._rules.models.data_types import DataType
12
- from cognite.neat.core._rules.models.entities import (
13
- ClassEntity,
10
+ from cognite.neat.core._data_model.models._base_unverified import (
11
+ UnverifiedComponent,
12
+ UnverifiedDataModel,
13
+ )
14
+ from cognite.neat.core._data_model.models.data_types import DataType
15
+ from cognite.neat.core._data_model.models.entities import (
16
+ ConceptEntity,
14
17
  MultiValueTypeInfo,
15
18
  UnknownEntity,
16
19
  load_value_type,
17
20
  )
18
21
  from cognite.neat.core._utils.rdf_ import uri_display_name
19
22
 
20
- from ._rules import (
21
- InformationClass,
22
- InformationMetadata,
23
- InformationProperty,
24
- InformationRules,
23
+ from ._verified import (
24
+ Concept,
25
+ ConceptualDataModel,
26
+ ConceptualMetadata,
27
+ ConceptualProperty,
25
28
  )
26
29
 
27
30
 
28
31
  @dataclass
29
- class InformationInputMetadata(InputComponent[InformationMetadata]):
32
+ class UnverifiedConceptualMetadata(UnverifiedComponent[ConceptualMetadata]):
30
33
  space: str
31
34
  external_id: str
32
35
  version: str
@@ -36,12 +39,11 @@ class InformationInputMetadata(InputComponent[InformationMetadata]):
36
39
  created: datetime | str | None = None
37
40
  updated: datetime | str | None = None
38
41
  physical: str | URIRef | None = None
39
- conceptual: str | URIRef | None = None
40
42
  source_id: str | URIRef | None = None
41
43
 
42
44
  @classmethod
43
- def _get_verified_cls(cls) -> type[InformationMetadata]:
44
- return InformationMetadata
45
+ def _get_verified_cls(cls) -> type[ConceptualMetadata]:
46
+ return ConceptualMetadata
45
47
 
46
48
  def dump(self, **kwargs: Any) -> dict[str, Any]:
47
49
  output = super().dump()
@@ -66,7 +68,7 @@ class InformationInputMetadata(InputComponent[InformationMetadata]):
66
68
  Unlike namespace, the identifier does not end with "/" or "#".
67
69
 
68
70
  """
69
- return DEFAULT_NAMESPACE[f"data-model/unverified/logical/{self.space}/{self.external_id}/{self.version}"]
71
+ return DEFAULT_NAMESPACE[f"data-model/unverified/conceptual/{self.space}/{self.external_id}/{self.version}"]
70
72
 
71
73
  @property
72
74
  def namespace(self) -> Namespace:
@@ -75,10 +77,10 @@ class InformationInputMetadata(InputComponent[InformationMetadata]):
75
77
 
76
78
 
77
79
  @dataclass
78
- class InformationInputProperty(InputComponent[InformationProperty]):
79
- class_: ClassEntity | str
80
+ class UnverifiedConceptualProperty(UnverifiedComponent[ConceptualProperty]):
81
+ concept: ConceptEntity | str
80
82
  property_: str
81
- value_type: DataType | ClassEntity | MultiValueTypeInfo | UnknownEntity | str
83
+ value_type: DataType | ConceptEntity | MultiValueTypeInfo | UnknownEntity | str
82
84
  name: str | None = None
83
85
  description: str | None = None
84
86
  min_count: int | None = None
@@ -91,65 +93,66 @@ class InformationInputProperty(InputComponent[InformationProperty]):
91
93
 
92
94
  # linking
93
95
  physical: str | URIRef | None = None
94
- conceptual: str | URIRef | None = None
95
96
 
96
97
  @classmethod
97
- def _get_verified_cls(cls) -> type[InformationProperty]:
98
- return InformationProperty
98
+ def _get_verified_cls(cls) -> type[ConceptualProperty]:
99
+ return ConceptualProperty
99
100
 
100
101
  def dump(self, default_prefix: str, **kwargs) -> dict[str, Any]: # type: ignore
101
102
  output = super().dump()
102
- output["Class"] = ClassEntity.load(self.class_, prefix=default_prefix)
103
+ output["Concept"] = ConceptEntity.load(self.concept, prefix=default_prefix)
103
104
  output["Value Type"] = load_value_type(self.value_type, default_prefix)
104
105
  return output
105
106
 
106
- def copy(self, update: dict[str, Any], default_prefix: str) -> "InformationInputProperty":
107
- return cast(InformationInputProperty, type(self)._load({**self.dump(default_prefix), **update}))
107
+ def copy(self, update: dict[str, Any], default_prefix: str) -> "UnverifiedConceptualProperty":
108
+ return cast(
109
+ UnverifiedConceptualProperty,
110
+ type(self)._load({**self.dump(default_prefix), **update}),
111
+ )
108
112
 
109
113
 
110
114
  @dataclass
111
- class InformationInputClass(InputComponent[InformationClass]):
112
- class_: ClassEntity | str
115
+ class UnverifiedConcept(UnverifiedComponent[Concept]):
116
+ concept: ConceptEntity | str
113
117
  name: str | None = None
114
118
  description: str | None = None
115
- implements: str | list[ClassEntity] | None = None
119
+ implements: str | list[ConceptEntity] | None = None
116
120
  instance_source: str | None = None
117
121
  neatId: str | URIRef | None = None
118
122
  # linking
119
123
  physical: str | URIRef | None = None
120
- conceptual: str | URIRef | None = None
121
124
 
122
125
  @classmethod
123
- def _get_verified_cls(cls) -> type[InformationClass]:
124
- return InformationClass
126
+ def _get_verified_cls(cls) -> type[Concept]:
127
+ return Concept
125
128
 
126
129
  @property
127
- def class_str(self) -> str:
128
- return str(self.class_)
130
+ def concept_str(self) -> str:
131
+ return str(self.concept)
129
132
 
130
133
  def dump(self, default_prefix: str, **kwargs) -> dict[str, Any]: # type: ignore
131
134
  output = super().dump()
132
- parent: list[ClassEntity] | None = None
135
+ parent: list[ConceptEntity] | None = None
133
136
  if isinstance(self.implements, str):
134
137
  self.implements = self.implements.strip()
135
- parent = [ClassEntity.load(parent, prefix=default_prefix) for parent in self.implements.split(",")]
138
+ parent = [ConceptEntity.load(parent, prefix=default_prefix) for parent in self.implements.split(",")]
136
139
  elif isinstance(self.implements, list):
137
- parent = [ClassEntity.load(parent_, prefix=default_prefix) for parent_ in self.implements]
138
- output["Class"] = ClassEntity.load(self.class_, prefix=default_prefix)
140
+ parent = [ConceptEntity.load(parent_, prefix=default_prefix) for parent_ in self.implements]
141
+ output["Concept"] = ConceptEntity.load(self.concept, prefix=default_prefix)
139
142
  output["Implements"] = parent
140
143
  return output
141
144
 
142
145
 
143
146
  @dataclass
144
- class InformationInputRules(InputRules[InformationRules]):
145
- metadata: InformationInputMetadata
146
- properties: list[InformationInputProperty] = field(default_factory=list)
147
- classes: list[InformationInputClass] = field(default_factory=list)
147
+ class UnverifiedConceptualDataModel(UnverifiedDataModel[ConceptualDataModel]):
148
+ metadata: UnverifiedConceptualMetadata
149
+ properties: list[UnverifiedConceptualProperty] = field(default_factory=list)
150
+ concepts: list[UnverifiedConcept] = field(default_factory=list)
148
151
  prefixes: dict[str, Namespace] | None = None
149
152
 
150
153
  @classmethod
151
- def _get_verified_cls(cls) -> type[InformationRules]:
152
- return InformationRules
154
+ def _get_verified_cls(cls) -> type[ConceptualDataModel]:
155
+ return ConceptualDataModel
153
156
 
154
157
  def dump(self) -> dict[str, Any]:
155
158
  default_prefix = self.metadata.prefix
@@ -157,7 +160,7 @@ class InformationInputRules(InputRules[InformationRules]):
157
160
  return dict(
158
161
  Metadata=self.metadata.dump(),
159
162
  Properties=[prop.dump(default_prefix) for prop in self.properties],
160
- Classes=[class_.dump(default_prefix) for class_ in self.classes],
163
+ Concepts=[concept.dump(default_prefix) for concept in self.concepts],
161
164
  Prefixes=self.prefixes,
162
165
  )
163
166
 
@@ -177,7 +180,7 @@ class InformationInputRules(InputRules[InformationRules]):
177
180
  "external_id": self.metadata.external_id,
178
181
  "space": self.metadata.space,
179
182
  "version": self.metadata.version,
180
- "classes": len(self.classes),
183
+ "concepts": len(self.concepts),
181
184
  "properties": len(self.properties),
182
185
  }
183
186