cognite-neat 0.87.6__py3-none-any.whl → 0.88.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cognite-neat might be problematic. Click here for more details.

Files changed (125) hide show
  1. cognite/neat/_version.py +1 -1
  2. cognite/neat/app/api/data_classes/rest.py +0 -19
  3. cognite/neat/app/api/explorer.py +6 -4
  4. cognite/neat/app/api/routers/crud.py +11 -21
  5. cognite/neat/app/api/routers/workflows.py +24 -94
  6. cognite/neat/graph/stores/_base.py +5 -0
  7. cognite/neat/rules/importers/_inference2rules.py +31 -35
  8. cognite/neat/workflows/steps/data_contracts.py +17 -43
  9. cognite/neat/workflows/steps/lib/current/graph_extractor.py +28 -24
  10. cognite/neat/workflows/steps/lib/current/graph_loader.py +4 -21
  11. cognite/neat/workflows/steps/lib/current/graph_store.py +18 -134
  12. cognite/neat/workflows/steps_registry.py +5 -7
  13. {cognite_neat-0.87.6.dist-info → cognite_neat-0.88.0.dist-info}/METADATA +1 -1
  14. {cognite_neat-0.87.6.dist-info → cognite_neat-0.88.0.dist-info}/RECORD +17 -125
  15. cognite/neat/app/api/routers/core.py +0 -91
  16. cognite/neat/app/api/routers/data_exploration.py +0 -336
  17. cognite/neat/app/api/routers/rules.py +0 -203
  18. cognite/neat/legacy/__init__.py +0 -0
  19. cognite/neat/legacy/graph/__init__.py +0 -3
  20. cognite/neat/legacy/graph/examples/Knowledge-Graph-Nordic44-dirty.xml +0 -20182
  21. cognite/neat/legacy/graph/examples/Knowledge-Graph-Nordic44.xml +0 -20163
  22. cognite/neat/legacy/graph/examples/__init__.py +0 -10
  23. cognite/neat/legacy/graph/examples/skos-capturing-sheet-wind-topics.xlsx +0 -0
  24. cognite/neat/legacy/graph/exceptions.py +0 -90
  25. cognite/neat/legacy/graph/extractors/__init__.py +0 -6
  26. cognite/neat/legacy/graph/extractors/_base.py +0 -14
  27. cognite/neat/legacy/graph/extractors/_dexpi.py +0 -44
  28. cognite/neat/legacy/graph/extractors/_graph_capturing_sheet.py +0 -403
  29. cognite/neat/legacy/graph/extractors/_mock_graph_generator.py +0 -361
  30. cognite/neat/legacy/graph/loaders/__init__.py +0 -23
  31. cognite/neat/legacy/graph/loaders/_asset_loader.py +0 -511
  32. cognite/neat/legacy/graph/loaders/_base.py +0 -67
  33. cognite/neat/legacy/graph/loaders/_exceptions.py +0 -85
  34. cognite/neat/legacy/graph/loaders/core/__init__.py +0 -0
  35. cognite/neat/legacy/graph/loaders/core/labels.py +0 -58
  36. cognite/neat/legacy/graph/loaders/core/models.py +0 -136
  37. cognite/neat/legacy/graph/loaders/core/rdf_to_assets.py +0 -1046
  38. cognite/neat/legacy/graph/loaders/core/rdf_to_relationships.py +0 -559
  39. cognite/neat/legacy/graph/loaders/rdf_to_dms.py +0 -309
  40. cognite/neat/legacy/graph/loaders/validator.py +0 -87
  41. cognite/neat/legacy/graph/models.py +0 -6
  42. cognite/neat/legacy/graph/stores/__init__.py +0 -13
  43. cognite/neat/legacy/graph/stores/_base.py +0 -400
  44. cognite/neat/legacy/graph/stores/_graphdb_store.py +0 -52
  45. cognite/neat/legacy/graph/stores/_memory_store.py +0 -43
  46. cognite/neat/legacy/graph/stores/_oxigraph_store.py +0 -151
  47. cognite/neat/legacy/graph/stores/_oxrdflib.py +0 -247
  48. cognite/neat/legacy/graph/stores/_rdf_to_graph.py +0 -42
  49. cognite/neat/legacy/graph/transformations/__init__.py +0 -0
  50. cognite/neat/legacy/graph/transformations/entity_matcher.py +0 -101
  51. cognite/neat/legacy/graph/transformations/query_generator/__init__.py +0 -3
  52. cognite/neat/legacy/graph/transformations/query_generator/sparql.py +0 -575
  53. cognite/neat/legacy/graph/transformations/transformer.py +0 -322
  54. cognite/neat/legacy/rules/__init__.py +0 -0
  55. cognite/neat/legacy/rules/analysis.py +0 -231
  56. cognite/neat/legacy/rules/examples/Rules-Nordic44-to-graphql.xlsx +0 -0
  57. cognite/neat/legacy/rules/examples/Rules-Nordic44.xlsx +0 -0
  58. cognite/neat/legacy/rules/examples/__init__.py +0 -18
  59. cognite/neat/legacy/rules/examples/power-grid-containers.yaml +0 -124
  60. cognite/neat/legacy/rules/examples/power-grid-example.xlsx +0 -0
  61. cognite/neat/legacy/rules/examples/power-grid-model.yaml +0 -224
  62. cognite/neat/legacy/rules/examples/rules-template.xlsx +0 -0
  63. cognite/neat/legacy/rules/examples/sheet2cdf-transformation-rules.xlsx +0 -0
  64. cognite/neat/legacy/rules/examples/skos-rules.xlsx +0 -0
  65. cognite/neat/legacy/rules/examples/source-to-solution-mapping-rules.xlsx +0 -0
  66. cognite/neat/legacy/rules/examples/wind-energy.owl +0 -1511
  67. cognite/neat/legacy/rules/exceptions.py +0 -2972
  68. cognite/neat/legacy/rules/exporters/__init__.py +0 -20
  69. cognite/neat/legacy/rules/exporters/_base.py +0 -45
  70. cognite/neat/legacy/rules/exporters/_core/__init__.py +0 -5
  71. cognite/neat/legacy/rules/exporters/_core/rules2labels.py +0 -24
  72. cognite/neat/legacy/rules/exporters/_rules2dms.py +0 -885
  73. cognite/neat/legacy/rules/exporters/_rules2excel.py +0 -213
  74. cognite/neat/legacy/rules/exporters/_rules2graphql.py +0 -183
  75. cognite/neat/legacy/rules/exporters/_rules2ontology.py +0 -524
  76. cognite/neat/legacy/rules/exporters/_rules2pydantic_models.py +0 -748
  77. cognite/neat/legacy/rules/exporters/_rules2rules.py +0 -105
  78. cognite/neat/legacy/rules/exporters/_rules2triples.py +0 -38
  79. cognite/neat/legacy/rules/exporters/_validation.py +0 -146
  80. cognite/neat/legacy/rules/importers/__init__.py +0 -22
  81. cognite/neat/legacy/rules/importers/_base.py +0 -66
  82. cognite/neat/legacy/rules/importers/_dict2rules.py +0 -158
  83. cognite/neat/legacy/rules/importers/_dms2rules.py +0 -194
  84. cognite/neat/legacy/rules/importers/_graph2rules.py +0 -308
  85. cognite/neat/legacy/rules/importers/_json2rules.py +0 -39
  86. cognite/neat/legacy/rules/importers/_owl2rules/__init__.py +0 -3
  87. cognite/neat/legacy/rules/importers/_owl2rules/_owl2classes.py +0 -239
  88. cognite/neat/legacy/rules/importers/_owl2rules/_owl2metadata.py +0 -260
  89. cognite/neat/legacy/rules/importers/_owl2rules/_owl2properties.py +0 -217
  90. cognite/neat/legacy/rules/importers/_owl2rules/_owl2rules.py +0 -290
  91. cognite/neat/legacy/rules/importers/_spreadsheet2rules.py +0 -45
  92. cognite/neat/legacy/rules/importers/_xsd2rules.py +0 -20
  93. cognite/neat/legacy/rules/importers/_yaml2rules.py +0 -39
  94. cognite/neat/legacy/rules/models/__init__.py +0 -5
  95. cognite/neat/legacy/rules/models/_base.py +0 -151
  96. cognite/neat/legacy/rules/models/raw_rules.py +0 -316
  97. cognite/neat/legacy/rules/models/rdfpath.py +0 -237
  98. cognite/neat/legacy/rules/models/rules.py +0 -1289
  99. cognite/neat/legacy/rules/models/tables.py +0 -9
  100. cognite/neat/legacy/rules/models/value_types.py +0 -118
  101. cognite/neat/legacy/workflows/examples/Export_DMS/workflow.yaml +0 -89
  102. cognite/neat/legacy/workflows/examples/Export_Rules_to_Ontology/workflow.yaml +0 -152
  103. cognite/neat/legacy/workflows/examples/Extract_DEXPI_Graph_and_Export_Rules/workflow.yaml +0 -139
  104. cognite/neat/legacy/workflows/examples/Extract_RDF_Graph_and_Generate_Assets/workflow.yaml +0 -270
  105. cognite/neat/legacy/workflows/examples/Import_DMS/workflow.yaml +0 -65
  106. cognite/neat/legacy/workflows/examples/Ontology_to_Data_Model/workflow.yaml +0 -116
  107. cognite/neat/legacy/workflows/examples/Validate_Rules/workflow.yaml +0 -67
  108. cognite/neat/legacy/workflows/examples/Validate_Solution_Model/workflow.yaml +0 -64
  109. cognite/neat/legacy/workflows/examples/Visualize_Data_Model_Using_Mock_Graph/workflow.yaml +0 -95
  110. cognite/neat/legacy/workflows/examples/Visualize_Semantic_Data_Model/workflow.yaml +0 -111
  111. cognite/neat/workflows/examples/Extract_RDF_Graph_and_Generate_Assets/workflow.yaml +0 -270
  112. cognite/neat/workflows/migration/__init__.py +0 -0
  113. cognite/neat/workflows/migration/steps.py +0 -91
  114. cognite/neat/workflows/migration/wf_manifests.py +0 -33
  115. cognite/neat/workflows/steps/lib/legacy/__init__.py +0 -7
  116. cognite/neat/workflows/steps/lib/legacy/graph_contextualization.py +0 -82
  117. cognite/neat/workflows/steps/lib/legacy/graph_extractor.py +0 -746
  118. cognite/neat/workflows/steps/lib/legacy/graph_loader.py +0 -606
  119. cognite/neat/workflows/steps/lib/legacy/graph_store.py +0 -307
  120. cognite/neat/workflows/steps/lib/legacy/graph_transformer.py +0 -58
  121. cognite/neat/workflows/steps/lib/legacy/rules_exporter.py +0 -511
  122. cognite/neat/workflows/steps/lib/legacy/rules_importer.py +0 -612
  123. {cognite_neat-0.87.6.dist-info → cognite_neat-0.88.0.dist-info}/LICENSE +0 -0
  124. {cognite_neat-0.87.6.dist-info → cognite_neat-0.88.0.dist-info}/WHEEL +0 -0
  125. {cognite_neat-0.87.6.dist-info → cognite_neat-0.88.0.dist-info}/entry_points.txt +0 -0
@@ -1,151 +0,0 @@
1
- import re
2
- import sys
3
- from typing import ClassVar
4
-
5
- from cognite.client.data_classes.data_modeling import ViewId
6
- from pydantic import BaseModel, ConfigDict, Field
7
- from rdflib import Literal, URIRef
8
-
9
- if sys.version_info >= (3, 11):
10
- from enum import StrEnum
11
- from typing import Self
12
- else:
13
- from backports.strenum import StrEnum
14
- from typing_extensions import Self
15
-
16
-
17
- class EntityTypes(StrEnum):
18
- subject = "subject"
19
- predicate = "predicate"
20
- object = "object"
21
- class_ = "class"
22
- property_ = "property"
23
- object_property = "ObjectProperty"
24
- data_property = "DatatypeProperty"
25
- annotation_property = "AnnotationProperty"
26
- data_value_type = "data_value_type"
27
- object_value_type = "object_value_type"
28
- view = "view"
29
- container = "container"
30
- undefined = "undefined"
31
-
32
-
33
- # ALLOWED
34
- ALLOWED_PATTERN = r"[^a-zA-Z0-9-_.]"
35
-
36
- # REGEX expressions
37
- PREFIX_REGEX = r"[a-zA-Z]+[a-zA-Z0-9-_.]*[a-zA-Z0-9]+"
38
- SUFFIX_REGEX = r"[a-zA-Z0-9-_.]+[a-zA-Z0-9]|[-_.]*[a-zA-Z0-9]+"
39
- VERSION_REGEX = r"[a-zA-Z0-9]([.a-zA-Z0-9_-]{0,41}[a-zA-Z0-9])"
40
-
41
- ENTITY_ID_REGEX = rf"{PREFIX_REGEX}:({SUFFIX_REGEX})"
42
- ENTITY_ID_REGEX_COMPILED = re.compile(rf"^(?P<prefix>{PREFIX_REGEX}):(?P<suffix>{SUFFIX_REGEX})$")
43
- VERSIONED_ENTITY_REGEX_COMPILED = re.compile(
44
- rf"^(?P<prefix>{PREFIX_REGEX}):(?P<suffix>{SUFFIX_REGEX})\(version=(?P<version>{VERSION_REGEX})\)$"
45
- )
46
-
47
- CLASS_ID_REGEX = rf"(?P<{EntityTypes.class_}>{ENTITY_ID_REGEX})"
48
- CLASS_ID_REGEX_COMPILED = re.compile(rf"^{CLASS_ID_REGEX}$")
49
-
50
- PROPERTY_ID_REGEX = rf"\((?P<{EntityTypes.property_}>{ENTITY_ID_REGEX})\)"
51
- VERSION_ID_REGEX = rf"\(version=(?P<version>{VERSION_REGEX})\)"
52
-
53
-
54
- class Entity(BaseModel):
55
- """Entity is a class or property in OWL/RDF sense."""
56
-
57
- model_config: ClassVar[ConfigDict] = ConfigDict(use_enum_values=True)
58
- prefix: str
59
- suffix: str
60
- type_: EntityTypes = Field(default=EntityTypes.undefined)
61
- name: str | None = None
62
- description: str | None = None
63
- version: str | None = None
64
-
65
- @property
66
- def id(self) -> str:
67
- return f"{self.prefix}:{self.suffix}"
68
-
69
- @property
70
- def versioned_id(self) -> str:
71
- if self.version:
72
- return f"{self.prefix}:{self.suffix}(version={self.version})"
73
- else:
74
- return self.id
75
-
76
- @property
77
- def space(self) -> str:
78
- """Returns entity space in CDF."""
79
- return self.prefix
80
-
81
- @property
82
- def external_id(self) -> str:
83
- """Returns entity external id in CDF."""
84
- return self.suffix
85
-
86
- def __repr__(self):
87
- return self.id
88
-
89
- @classmethod
90
- def from_string(cls, entity_string: str, base_prefix: str | None = None, **kwargs) -> Self:
91
- if result := VERSIONED_ENTITY_REGEX_COMPILED.match(entity_string):
92
- return cls(
93
- prefix=result.group("prefix"),
94
- suffix=result.group("suffix"),
95
- name=result.group("suffix"),
96
- version=result.group("version"),
97
- **kwargs,
98
- )
99
- elif result := ENTITY_ID_REGEX_COMPILED.match(entity_string):
100
- return cls(
101
- prefix=result.group("prefix"), suffix=result.group("suffix"), name=result.group("suffix"), **kwargs
102
- )
103
- elif base_prefix and re.match(SUFFIX_REGEX, entity_string) and re.match(PREFIX_REGEX, base_prefix):
104
- return cls(prefix=base_prefix, suffix=entity_string, name=entity_string, **kwargs)
105
- else:
106
- raise ValueError(f"{cls.__name__} is expected to be prefix:suffix, got {entity_string}")
107
-
108
- @classmethod
109
- def from_list(cls, entity_strings: list[str], base_prefix: str | None = None, **kwargs) -> list[Self]:
110
- return [
111
- cls.from_string(entity_string=entity_string, base_prefix=base_prefix, **kwargs)
112
- for entity_string in entity_strings
113
- ]
114
-
115
-
116
- class ParentClass(Entity):
117
- type_: EntityTypes = EntityTypes.class_
118
-
119
- @property
120
- def view_id(self) -> ViewId:
121
- return ViewId(space=self.space, external_id=self.external_id, version=self.version)
122
-
123
- @classmethod
124
- def from_view_id(cls, view_id: ViewId) -> Self:
125
- return cls(prefix=view_id.space, suffix=view_id.external_id, version=view_id.version)
126
-
127
-
128
- class ContainerEntity(Entity):
129
- type_: EntityTypes = EntityTypes.container
130
-
131
-
132
- class ViewEntity(Entity):
133
- type_: EntityTypes = EntityTypes.view
134
-
135
-
136
- class Triple(BaseModel):
137
- model_config: ClassVar[ConfigDict] = ConfigDict(
138
- populate_by_name=True, arbitrary_types_allowed=True, strict=False, extra="allow"
139
- )
140
-
141
- subject: str | URIRef | Entity
142
- predicate: str | URIRef | Entity
143
- object: str | URIRef | Literal | Entity | None = None
144
- optional: bool = Field(
145
- description="Indicates whether a triple is optional, used when building SPARQL query",
146
- default=False,
147
- )
148
-
149
- @classmethod
150
- def from_rdflib_triple(cls, triple: tuple[URIRef, URIRef, URIRef | Literal]) -> Self:
151
- return cls(subject=triple[0], predicate=triple[1], object=triple[2])
@@ -1,316 +0,0 @@
1
- import logging
2
- import warnings
3
- from collections.abc import Hashable
4
- from typing import Any, cast, no_type_check
5
- from warnings import warn
6
-
7
- import pandas as pd
8
- from pydantic import field_validator
9
- from pydantic_core import ErrorDetails, ValidationError
10
- from rdflib import Namespace
11
-
12
- from cognite.neat.constants import get_default_prefixes
13
- from cognite.neat.exceptions import wrangle_warnings
14
-
15
- # rules model and model components:
16
- from cognite.neat.legacy.rules.models.rules import (
17
- Class,
18
- Metadata,
19
- Property,
20
- RuleModel,
21
- Rules,
22
- )
23
- from cognite.neat.legacy.rules.models.tables import Tables
24
-
25
- # importers:
26
- from cognite.neat.rules import exceptions
27
- from cognite.neat.utils.auxiliary import generate_exception_report
28
-
29
- __all__ = ["RawRules"]
30
-
31
-
32
- class RawRules(RuleModel):
33
- """
34
- RawRules represent invalidated form of Rules, which is a core concept in `neat`.
35
- RawRules are used as staging area for Rules, and are often used when importing rules
36
- from sources other than Excel rules, e.g. from a json schema or owl ontology. Often
37
- these sources are not validated, and are missing information to be able to create
38
- a valid Rules object.
39
-
40
- Args:
41
- Metadata: Data model metadata
42
- classes: Classes defined in the data model
43
- properties: Class properties defined in the data model with accompanying transformation rules
44
- to transform data from source to target representation
45
- prefixes: Prefixes used in the data model. Defaults to internal set of prefixes
46
- instances: Instances defined in the data model. Defaults to None
47
- """
48
-
49
- Metadata: pd.DataFrame
50
- Classes: pd.DataFrame
51
- Properties: pd.DataFrame
52
- Prefixes: pd.DataFrame = pd.DataFrame()
53
- Instances: pd.DataFrame = pd.DataFrame()
54
- importer_type: str = "RawTablesImporter"
55
-
56
- @field_validator("Metadata")
57
- def has_metadata_mandatory_rows(cls, v: pd.DataFrame):
58
- given_rows = set(v.iloc[:, 0].values)
59
- mandatory_rows = Metadata.mandatory_fields()
60
- mandatory_rows_alias = Metadata.mandatory_fields(use_alias=True)
61
-
62
- if not (mandatory_rows.issubset(given_rows) or mandatory_rows_alias.issubset(given_rows)):
63
- missing_rows = mandatory_rows_alias.difference(given_rows)
64
- raise exceptions.MetadataSheetMissingMandatoryFields(missing_rows).to_pydantic_custom_error()
65
- return v
66
-
67
- @field_validator("Classes")
68
- def has_classes_mandatory_columns(cls, v):
69
- given_columns = set(v.columns)
70
- mandatory_columns = Class.mandatory_fields()
71
- mandatory_columns_alias = Class.mandatory_fields(use_alias=True)
72
-
73
- if not (mandatory_columns.issubset(given_columns) or mandatory_columns_alias.issubset(given_columns)):
74
- missing_columns = mandatory_columns_alias.difference(given_columns)
75
- raise exceptions.ClassesSheetMissingMandatoryColumns(missing_columns).to_pydantic_custom_error()
76
- return v
77
-
78
- @field_validator("Properties")
79
- def has_properties_mandatory_columns(cls, v):
80
- given_columns = set(v.columns)
81
- mandatory_columns = Property.mandatory_fields()
82
- mandatory_columns_alias = Property.mandatory_fields(use_alias=True)
83
-
84
- if not (mandatory_columns.issubset(given_columns) or mandatory_columns_alias.issubset(given_columns)):
85
- missing_columns = mandatory_columns_alias.difference(given_columns)
86
- raise exceptions.PropertiesSheetMissingMandatoryColumns(missing_columns).to_pydantic_custom_error()
87
- return v
88
-
89
- @field_validator("Prefixes")
90
- def has_prefixes_mandatory_columns(cls, v):
91
- given_columns = set(v.columns)
92
- mandatory_columns = {"Prefix", "URI"}
93
-
94
- if not mandatory_columns.issubset(given_columns):
95
- missing_columns = mandatory_columns.difference(given_columns)
96
- raise exceptions.PrefixesSheetMissingMandatoryColumns(missing_columns).to_pydantic_custom_error()
97
- return v
98
-
99
- @field_validator("Instances")
100
- def has_instances_mandatory_columns(cls, v):
101
- given_columns = set(v.columns)
102
- mandatory_columns = {"Instance", "Property", "Value"}
103
-
104
- if not mandatory_columns.issubset(given_columns):
105
- missing_columns = mandatory_columns.difference(given_columns)
106
- raise exceptions.InstancesSheetMissingMandatoryColumns(missing_columns).to_pydantic_custom_error()
107
- return v
108
-
109
- @staticmethod
110
- def _drop_non_string_columns(df: pd.DataFrame) -> pd.DataFrame:
111
- """Drop non-string columns as this can cause issue when loading rules
112
-
113
- Args:
114
- df: data frame
115
-
116
- Returns:
117
- dataframe with removed non string columns
118
- """
119
- columns = [column for column in df.columns[df.columns.notna()] if isinstance(column, str)]
120
-
121
- return df[columns]
122
-
123
- # mypy complains "RawRules" has incompatible type "**dict[str, DataFrame]"; expected "set[str]" , which is wrong!
124
- @no_type_check
125
- @classmethod
126
- def from_tables(cls, tables: dict[str, pd.DataFrame], importer_type: str = "RawTablesImporter") -> "RawRules":
127
- """Create RawRules from raw tables.
128
-
129
- Args:
130
- tables: Tables to be converted to RawRules
131
-
132
- Returns:
133
- Instance of RawRules
134
- """
135
-
136
- expected_tables = cls.mandatory_fields()
137
-
138
- # Validate raw tables
139
- if missing_tables := (expected_tables - set(tables)):
140
- raise exceptions.SourceObjectDoesNotProduceMandatorySheets(missing_tables)
141
-
142
- tables_dict: dict[str, pd.DataFrame] = {
143
- Tables.metadata: tables[Tables.metadata],
144
- Tables.classes: cls._drop_non_string_columns(tables[Tables.classes]),
145
- Tables.properties: cls._drop_non_string_columns(tables[Tables.properties]),
146
- }
147
-
148
- if Tables.prefixes in tables:
149
- tables_dict[Tables.prefixes] = cls._drop_non_string_columns(tables[Tables.prefixes])
150
- if Tables.instances in tables:
151
- tables_dict[Tables.instances] = cls._drop_non_string_columns(tables[Tables.instances])
152
-
153
- return cls(
154
- **tables_dict,
155
- importer_type=importer_type,
156
- )
157
-
158
- # mypy unsatisfied with overload , tried all combination and gave up
159
- @no_type_check
160
- def to_rules(
161
- self,
162
- return_report: bool = False,
163
- skip_validation: bool = False,
164
- validators_to_skip: set[str] | None = None,
165
- ) -> tuple[Rules | None, list[ErrorDetails] | None, list | None] | Rules:
166
- """Validates RawRules instances and returns Rules instance.
167
-
168
- Args:
169
- return_report: To return validation report. Defaults to False.
170
- skip_validation: Bypasses Rules validation. Defaults to False.
171
- validators_to_skip: List of validators to skip. Defaults to None.
172
-
173
- Returns:
174
- Instance of `Rules`, which can be validated or not validated based on
175
- `skip_validation` flag, and optional list of errors and warnings if
176
- `return_report` is set to True.
177
-
178
- !!! Note
179
- `skip_validation` flag should be only used for purpose when `Rules` object
180
- is exported to an Excel file. Do not use this flag for any other purpose!
181
- """
182
-
183
- rules_dict = _raw_tables_to_rules_dict(self, validators_to_skip)
184
- if skip_validation:
185
- return _to_invalidated_rules(rules_dict)
186
- else:
187
- return _to_validated_rules(rules_dict, return_report)
188
-
189
- def validate_rules(self) -> str | None:
190
- _, errors, warnings_ = self.to_rules(return_report=True, skip_validation=False)
191
-
192
- report = ""
193
- if errors:
194
- warnings.warn(
195
- exceptions.RulesHasErrors(importer_type=self.importer_type).message,
196
- category=exceptions.RulesHasErrors,
197
- stacklevel=2,
198
- )
199
- report = generate_exception_report(cast(list[ErrorDetails], errors), "Errors")
200
-
201
- if warnings_:
202
- warnings.warn(
203
- exceptions.RulesHasWarnings(importer_type=self.importer_type).message,
204
- category=exceptions.RulesHasWarnings,
205
- stacklevel=2,
206
- )
207
- report += generate_exception_report(cast(list[dict], warnings_), "Warnings")
208
-
209
- return report if report else None
210
-
211
-
212
- def _to_validated_rules(
213
- rules_dict: dict, return_report: bool = False
214
- ) -> tuple[Rules | None, list[ErrorDetails] | None, list[dict] | None] | Rules:
215
- validation_warnings = []
216
- try:
217
- with warnings.catch_warnings(record=True) as validation_warnings:
218
- rules = Rules(**rules_dict)
219
- return (rules, None, wrangle_warnings(validation_warnings)) if return_report else rules
220
-
221
- except ValidationError as e:
222
- validation_errors = e.errors()
223
- if return_report:
224
- return None, validation_errors, wrangle_warnings(validation_warnings)
225
- else:
226
- raise e
227
-
228
-
229
- def _to_invalidated_rules(rules_dict: dict) -> Rules:
230
- args = {
231
- "metadata": Metadata.model_construct(**rules_dict["metadata"]),
232
- "classes": {
233
- class_: Class.model_construct(**definition) for class_, definition in rules_dict["classes"].items()
234
- },
235
- "properties": {
236
- property_: Property.model_construct(**definition)
237
- for property_, definition in rules_dict["properties"].items()
238
- },
239
- "prefixes": rules_dict["prefixes"],
240
- }
241
-
242
- return cast(Rules, Rules.model_construct(**args))
243
-
244
-
245
- def _raw_tables_to_rules_dict(raw_tables: RawRules, validators_to_skip: set | None = None) -> dict[str, Any]:
246
- """Converts raw tables to a dictionary of rules."""
247
- rules_dict: dict[str, Any] = {
248
- "metadata": _metadata_table2dict(raw_tables.Metadata),
249
- "classes": _classes_table2dict(raw_tables.Classes),
250
- "properties": _properties_table2dict(raw_tables.Properties),
251
- "prefixes": (
252
- get_default_prefixes() if raw_tables.Prefixes.empty else _prefixes_table2dict(raw_tables.Prefixes)
253
- ),
254
- }
255
-
256
- rules_dict["instances"] = (
257
- None
258
- if raw_tables.Instances.empty
259
- else _instances_table2dict(raw_tables.Instances, rules_dict["metadata"], rules_dict["prefixes"])
260
- )
261
-
262
- if validators_to_skip:
263
- rules_dict["validators_to_skip"] = validators_to_skip
264
- rules_dict["metadata"]["validators_to_skip"] = validators_to_skip
265
- for class_ in rules_dict["classes"].keys():
266
- rules_dict["classes"][class_]["validators_to_skip"] = validators_to_skip
267
- for property_ in rules_dict["properties"].keys():
268
- rules_dict["properties"][property_]["validators_to_skip"] = validators_to_skip
269
-
270
- return rules_dict
271
-
272
-
273
- def _metadata_table2dict(meta_df: pd.DataFrame) -> dict[str, Any]:
274
- assert len(meta_df.columns) == 2
275
- col1, col2 = meta_df.columns
276
- metadata_dict = dict(zip(meta_df[col1], meta_df[col2], strict=True))
277
- metadata_dict["source"] = meta_df.source if "source" in dir(meta_df) else None
278
- if "namespace" in metadata_dict:
279
- metadata_dict["namespace"] = Namespace(metadata_dict["namespace"])
280
- return metadata_dict
281
-
282
-
283
- def _classes_table2dict(
284
- classes_df: pd.DataFrame,
285
- ) -> dict[Any | None, dict[Hashable, Any]]:
286
- return {class_.get("Class"): class_ for class_ in classes_df.to_dict(orient="records")}
287
-
288
-
289
- def _properties_table2dict(
290
- properties_df: pd.DataFrame,
291
- ) -> dict[str, dict[Hashable, Any]]:
292
- return {f"row {i+3}": property_ for i, property_ in enumerate(properties_df.to_dict(orient="records"))}
293
-
294
-
295
- def _prefixes_table2dict(prefix_df: pd.DataFrame) -> dict[str, Namespace]:
296
- return {row["Prefix"]: Namespace(row["URI"]) for i, row in prefix_df.iterrows()}
297
-
298
-
299
- def _instances_table2dict(
300
- instances_df: pd.DataFrame, metadata: dict[str, Any], prefixes: dict[str, Namespace]
301
- ) -> list[dict] | None:
302
- if ("prefix" not in metadata and "namespace" not in metadata) or "namespace" not in metadata:
303
- logging.warning(exceptions.MissingDataModelPrefixOrNamespace().message)
304
- warn(exceptions.MissingDataModelPrefixOrNamespace().message, stacklevel=2)
305
- return None
306
-
307
- prefix = metadata["prefix"] if "prefix" in metadata else metadata["space"]
308
- prefixes[prefix] = metadata["namespace"]
309
-
310
- instances = []
311
- for _, row in instances_df.iterrows():
312
- row_as_dict = row.to_dict()
313
- row_as_dict["namespace"] = metadata["namespace"]
314
- row_as_dict["prefixes"] = prefixes
315
- instances.append(row_as_dict)
316
- return instances
@@ -1,237 +0,0 @@
1
- """ """
2
-
3
- import re
4
- import sys
5
- from collections import Counter
6
- from typing import Literal
7
-
8
- from pydantic import BaseModel, field_validator
9
-
10
- from cognite.neat.legacy.rules import exceptions
11
-
12
- from ._base import (
13
- CLASS_ID_REGEX,
14
- CLASS_ID_REGEX_COMPILED,
15
- ENTITY_ID_REGEX,
16
- PROPERTY_ID_REGEX,
17
- SUFFIX_REGEX,
18
- Entity,
19
- EntityTypes,
20
- )
21
-
22
- if sys.version_info >= (3, 11):
23
- from enum import StrEnum
24
- from typing import Self
25
- else:
26
- from backports.strenum import StrEnum
27
- from typing_extensions import Self
28
-
29
-
30
- class TransformationRuleType(StrEnum):
31
- rdfpath = "rdfpath"
32
- rawlookup = "rawlookup"
33
- sparql = "sparql"
34
-
35
-
36
- class Lookup(StrEnum):
37
- table = "table"
38
- key = "key"
39
- value = "value" # type: ignore
40
-
41
-
42
- # traversal direction
43
- DIRECTION_REGEX = r"(?P<direction>(->|<-))"
44
-
45
- # steps
46
- STEP_REGEX = rf"((->|<-){CLASS_ID_REGEX}({PROPERTY_ID_REGEX})?)"
47
- STEP_REGEX_COMPILED = re.compile(STEP_REGEX)
48
- STEP_CLASS_REGEX_COMPILED = re.compile(rf"(^{DIRECTION_REGEX}{CLASS_ID_REGEX})$")
49
- STEP_CLASS_AND_PROPERTY_REGEX_COMPILED = re.compile(rf"(^{DIRECTION_REGEX}{CLASS_ID_REGEX}{PROPERTY_ID_REGEX}$)")
50
-
51
-
52
- _traversal = "traversal"
53
- ORIGIN_REGEX = rf"(?P<origin>{ENTITY_ID_REGEX})"
54
-
55
- HOP_REGEX_COMPILED = re.compile(rf"^{ORIGIN_REGEX}(?P<{_traversal}>{STEP_REGEX}+)$")
56
-
57
- # grabbing specific property for a class, property can be either object, annotation or data property
58
- SINGLE_PROPERTY_REGEX_COMPILED = re.compile(rf"^{CLASS_ID_REGEX}{PROPERTY_ID_REGEX}$")
59
-
60
- # grabbing all properties for a class
61
- ALL_PROPERTIES_REGEX_COMPILED = re.compile(rf"^{CLASS_ID_REGEX}\(\*\)$")
62
-
63
- ALL_TRAVERSAL_REGEX_COMPILED = (
64
- rf"({CLASS_ID_REGEX}\(\*\)|{CLASS_ID_REGEX}{PROPERTY_ID_REGEX}|{ORIGIN_REGEX}(?P<{_traversal}>{STEP_REGEX}+))"
65
- )
66
-
67
- TABLE_REGEX_COMPILED = re.compile(
68
- rf"^(?P<{Lookup.table}>{SUFFIX_REGEX})\((?P<{Lookup.key}>{SUFFIX_REGEX}),\s*(?P<{Lookup.value}>{SUFFIX_REGEX})\)$"
69
- )
70
-
71
-
72
- StepDirection = Literal["source", "target", "origin"]
73
- _direction_by_symbol: dict[str, StepDirection] = {"->": "target", "<-": "source"}
74
-
75
-
76
- class Step(BaseModel):
77
- class_: Entity
78
- property: Entity | None = None # only terminal step has property
79
- direction: StepDirection
80
-
81
- @classmethod
82
- def from_string(cls, raw: str, **kwargs) -> Self:
83
- if result := STEP_CLASS_AND_PROPERTY_REGEX_COMPILED.match(raw):
84
- return cls(
85
- class_=Entity.from_string(result.group(EntityTypes.class_), type_="class"),
86
- property=Entity.from_string(result.group(EntityTypes.property_), type_="property"),
87
- direction=_direction_by_symbol[result.group("direction")],
88
- **kwargs,
89
- )
90
- elif result := STEP_CLASS_REGEX_COMPILED.match(raw):
91
- return cls(
92
- class_=Entity.from_string(result.group(EntityTypes.class_)),
93
- direction=_direction_by_symbol[result.group("direction")],
94
- ) # type: ignore
95
- msg = f"Invalid step {raw}, expected in one of the following forms:"
96
- msg += " ->prefix:suffix, <-prefix:suffix, ->prefix:suffix(prefix:suffix) or <-prefix:suffix(prefix:suffix)"
97
- raise ValueError(msg)
98
-
99
-
100
- class Traversal(BaseModel):
101
- class_: Entity
102
-
103
-
104
- class SingleProperty(Traversal):
105
- property: Entity
106
-
107
- @classmethod
108
- def from_string(cls, class_: str, property_: str) -> Self:
109
- return cls(
110
- class_=Entity.from_string(class_, type_="class"), property=Entity.from_string(property_, type_="property")
111
- )
112
-
113
-
114
- class AllReferences(Traversal):
115
- @classmethod
116
- def from_string(cls, class_: str) -> Self:
117
- return cls(class_=Entity.from_string(class_, type_="class"))
118
-
119
-
120
- class AllProperties(Traversal):
121
- @classmethod
122
- def from_string(cls, class_: str) -> Self:
123
- return cls(class_=Entity.from_string(class_, type_="class"))
124
-
125
-
126
- class Origin(BaseModel):
127
- class_: Entity
128
-
129
- @field_validator("class_", mode="before")
130
- def process_if_string(cls, value):
131
- return Entity.from_string(value) if isinstance(value, str) else value
132
-
133
-
134
- class Hop(Traversal):
135
- """Multi or single hop traversal through graph"""
136
-
137
- traversal: list[Step]
138
-
139
- @classmethod
140
- def from_string(cls, class_: str, traversal: str | list[Step]) -> Self:
141
- return cls(
142
- class_=Entity.from_string(class_),
143
- traversal=(
144
- [Step.from_string(result[0]) for result in STEP_REGEX_COMPILED.findall(traversal)]
145
- if isinstance(traversal, str)
146
- else traversal
147
- ),
148
- )
149
-
150
-
151
- class TableLookup(BaseModel):
152
- name: str
153
- key: str
154
- value: str
155
-
156
-
157
- class Rule(BaseModel):
158
- pass
159
-
160
-
161
- class Query(BaseModel):
162
- query: str
163
-
164
-
165
- class RDFPath(Rule):
166
- traversal: Traversal | Query
167
-
168
-
169
- class RawLookup(RDFPath):
170
- table: TableLookup
171
-
172
-
173
- class SPARQLQuery(RDFPath):
174
- traversal: Query
175
-
176
-
177
- def parse_traversal(raw: str) -> AllReferences | AllProperties | SingleProperty | Hop:
178
- if result := CLASS_ID_REGEX_COMPILED.match(raw):
179
- return AllReferences.from_string(class_=result.group(EntityTypes.class_))
180
- elif result := ALL_PROPERTIES_REGEX_COMPILED.match(raw):
181
- return AllProperties.from_string(class_=result.group(EntityTypes.class_))
182
- elif result := SINGLE_PROPERTY_REGEX_COMPILED.match(raw):
183
- return SingleProperty.from_string(
184
- class_=result.group(EntityTypes.class_), property_=result.group(EntityTypes.property_)
185
- )
186
- elif result := HOP_REGEX_COMPILED.match(raw):
187
- return Hop.from_string(class_=result.group("origin"), traversal=result.group(_traversal))
188
- else:
189
- raise exceptions.NotValidRDFPath(raw).to_pydantic_custom_error()
190
-
191
-
192
- def parse_table_lookup(raw: str) -> TableLookup:
193
- if result := TABLE_REGEX_COMPILED.match(raw):
194
- return TableLookup(
195
- name=result.group(Lookup.table), key=result.group(Lookup.key), value=result.group(Lookup.value)
196
- )
197
- raise exceptions.NotValidTableLookUp(raw).to_pydantic_custom_error()
198
-
199
-
200
- def parse_rule(rule_raw: str, rule_type: TransformationRuleType | None) -> RDFPath:
201
- match rule_type:
202
- case TransformationRuleType.rdfpath:
203
- rule_raw = rule_raw.replace(" ", "")
204
- return RDFPath(traversal=parse_traversal(rule_raw))
205
- case TransformationRuleType.rawlookup:
206
- rule_raw = rule_raw.replace(" ", "")
207
- if Counter(rule_raw).get("|") != 1:
208
- raise exceptions.NotValidRAWLookUp(rule_raw).to_pydantic_custom_error()
209
- traversal, table_lookup = rule_raw.split("|")
210
- return RawLookup(traversal=parse_traversal(traversal), table=parse_table_lookup(table_lookup))
211
- case TransformationRuleType.sparql:
212
- return SPARQLQuery(traversal=Query(query=rule_raw))
213
- case None:
214
- raise ValueError("Rule type must be specified")
215
-
216
-
217
- def is_valid_rule(rule_type: TransformationRuleType, rule_raw: str) -> bool:
218
- is_valid_rule = {TransformationRuleType.rdfpath: is_rdfpath, TransformationRuleType.rawlookup: is_rawlookup}[
219
- rule_type
220
- ]
221
- return is_valid_rule(rule_raw)
222
-
223
-
224
- def is_rdfpath(raw: str) -> bool:
225
- try:
226
- parse_traversal(raw)
227
- except ValueError:
228
- return False
229
- return True
230
-
231
-
232
- def is_rawlookup(raw: str) -> bool:
233
- try:
234
- parse_rule(raw, TransformationRuleType.rawlookup)
235
- except ValueError:
236
- return False
237
- return True