cognite-neat 0.100.0__py3-none-any.whl → 0.101.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_client/_api/data_modeling_loaders.py +28 -8
- cognite/neat/_graph/loaders/_rdf2dms.py +0 -1
- cognite/neat/_graph/transformers/__init__.py +5 -0
- cognite/neat/_graph/transformers/_iodd.py +9 -4
- cognite/neat/_graph/transformers/_prune_graph.py +118 -61
- cognite/neat/_rules/exporters/_rules2dms.py +38 -16
- cognite/neat/_rules/models/_base_rules.py +13 -9
- cognite/neat/_rules/models/dms/_rules.py +111 -39
- cognite/neat/_rules/models/information/_rules.py +50 -17
- cognite/neat/_session/_base.py +18 -0
- cognite/neat/_session/_read.py +3 -3
- cognite/neat/_session/_to.py +3 -3
- cognite/neat/_utils/reader/__init__.py +2 -2
- cognite/neat/_utils/reader/_base.py +40 -35
- cognite/neat/_utils/text.py +12 -0
- cognite/neat/_version.py +1 -1
- cognite_neat-0.101.0.dist-info/METADATA +113 -0
- {cognite_neat-0.100.0.dist-info → cognite_neat-0.101.0.dist-info}/RECORD +21 -21
- cognite_neat-0.100.0.dist-info/METADATA +0 -215
- {cognite_neat-0.100.0.dist-info → cognite_neat-0.101.0.dist-info}/LICENSE +0 -0
- {cognite_neat-0.100.0.dist-info → cognite_neat-0.101.0.dist-info}/WHEEL +0 -0
- {cognite_neat-0.100.0.dist-info → cognite_neat-0.101.0.dist-info}/entry_points.txt +0 -0
|
@@ -94,20 +94,60 @@ def _metadata(context: Any) -> DMSMetadata | None:
|
|
|
94
94
|
|
|
95
95
|
|
|
96
96
|
class DMSProperty(SheetRow):
|
|
97
|
-
view: ViewEntityType = Field(alias="View")
|
|
98
|
-
view_property: DmsPropertyType = Field(alias="View Property")
|
|
99
|
-
name: str | None = Field(alias="Name", default=None)
|
|
100
|
-
description: str | None = Field(alias="Description", default=None)
|
|
101
|
-
connection: Literal["direct"] | ReverseConnectionEntity | EdgeEntity | None = Field(
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
97
|
+
view: ViewEntityType = Field(alias="View", description="The property identifier.")
|
|
98
|
+
view_property: DmsPropertyType = Field(alias="View Property", description="The ViewId this property belongs to")
|
|
99
|
+
name: str | None = Field(alias="Name", default=None, description="Human readable name of the property")
|
|
100
|
+
description: str | None = Field(alias="Description", default=None, description="Short description of the property")
|
|
101
|
+
connection: Literal["direct"] | ReverseConnectionEntity | EdgeEntity | None = Field(
|
|
102
|
+
None,
|
|
103
|
+
alias="Connection",
|
|
104
|
+
description="nly applies to connection between views. "
|
|
105
|
+
"It specify how the connection should be implemented in CDF.",
|
|
106
|
+
)
|
|
107
|
+
value_type: DataType | ViewEntity | DMSUnknownEntity = Field(
|
|
108
|
+
alias="Value Type",
|
|
109
|
+
description="Value type that the property can hold. "
|
|
110
|
+
"It takes either subset of CDF primitive types or a View id",
|
|
111
|
+
)
|
|
112
|
+
nullable: bool | None = Field(
|
|
113
|
+
default=None,
|
|
114
|
+
alias="Nullable",
|
|
115
|
+
description="Used to indicate whether the property is required or not. Only applies to primitive type.",
|
|
116
|
+
)
|
|
117
|
+
immutable: bool | None = Field(
|
|
118
|
+
default=None,
|
|
119
|
+
alias="Immutable",
|
|
120
|
+
description="sed to indicate whether the property is can only be set once. Only applies to primitive type.",
|
|
121
|
+
)
|
|
122
|
+
is_list: bool | None = Field(
|
|
123
|
+
default=None,
|
|
124
|
+
alias="Is List",
|
|
125
|
+
description="Used to indicate whether the property holds single or multiple values (list). "
|
|
126
|
+
"Only applies to primitive types.",
|
|
127
|
+
)
|
|
128
|
+
default: str | int | dict | None = Field(
|
|
129
|
+
None, alias="Default", description="Specifies default value for the property."
|
|
130
|
+
)
|
|
131
|
+
container: ContainerEntityType | None = Field(
|
|
132
|
+
None,
|
|
133
|
+
alias="Container",
|
|
134
|
+
description="Specifies container where the property is stored. Only applies to primitive type.",
|
|
135
|
+
)
|
|
136
|
+
container_property: DmsPropertyType | None = Field(
|
|
137
|
+
None,
|
|
138
|
+
alias="Container Property",
|
|
139
|
+
description="Specifies property in the container where the property is stored. Only applies to primitive type.",
|
|
140
|
+
)
|
|
141
|
+
index: StrListType | None = Field(
|
|
142
|
+
None,
|
|
143
|
+
alias="Index",
|
|
144
|
+
description="The names of the indexes (comma separated) that should be created for the property.",
|
|
145
|
+
)
|
|
146
|
+
constraint: StrListType | None = Field(
|
|
147
|
+
None,
|
|
148
|
+
alias="Constraint",
|
|
149
|
+
description="The names of the uniquness (comma separated) that should be created for the property.",
|
|
150
|
+
)
|
|
111
151
|
logical: URIRefType | None = Field(
|
|
112
152
|
None,
|
|
113
153
|
alias="Logical",
|
|
@@ -192,11 +232,21 @@ class DMSProperty(SheetRow):
|
|
|
192
232
|
|
|
193
233
|
|
|
194
234
|
class DMSContainer(SheetRow):
|
|
195
|
-
container: ContainerEntityType = Field(
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
235
|
+
container: ContainerEntityType = Field(
|
|
236
|
+
alias="Container", description="Container id, strongly advised to PascalCase usage."
|
|
237
|
+
)
|
|
238
|
+
name: str | None = Field(
|
|
239
|
+
alias="Name", default=None, description="Human readable name of the container being defined."
|
|
240
|
+
)
|
|
241
|
+
description: str | None = Field(
|
|
242
|
+
alias="Description", default=None, description="Short description of the node being defined."
|
|
243
|
+
)
|
|
244
|
+
constraint: ContainerEntityList | None = Field(
|
|
245
|
+
None, alias="Constraint", description="List of required (comma separated) constraints for the container"
|
|
246
|
+
)
|
|
247
|
+
used_for: Literal["node", "edge", "all"] | None = Field(
|
|
248
|
+
"all", alias="Used For", description=" Whether the container is used for nodes, edges or all."
|
|
249
|
+
)
|
|
200
250
|
|
|
201
251
|
def _identifier(self) -> tuple[Hashable, ...]:
|
|
202
252
|
return (self.container,)
|
|
@@ -240,12 +290,22 @@ class DMSContainer(SheetRow):
|
|
|
240
290
|
|
|
241
291
|
|
|
242
292
|
class DMSView(SheetRow):
|
|
243
|
-
view: ViewEntityType = Field(alias="View")
|
|
244
|
-
name: str | None = Field(alias="Name", default=None)
|
|
245
|
-
description: str | None = Field(
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
293
|
+
view: ViewEntityType = Field(alias="View", description="View id, strongly advised to PascalCase usage.")
|
|
294
|
+
name: str | None = Field(alias="Name", default=None, description="Human readable name of the view being defined.")
|
|
295
|
+
description: str | None = Field(
|
|
296
|
+
alias="Description", default=None, description="Short description of the view being defined "
|
|
297
|
+
)
|
|
298
|
+
implements: ViewEntityList | None = Field(
|
|
299
|
+
None,
|
|
300
|
+
alias="Implements",
|
|
301
|
+
description="List of parent view ids (comma separated) which the view being defined implements.",
|
|
302
|
+
)
|
|
303
|
+
filter_: HasDataFilter | NodeTypeFilter | RawFilter | None = Field(
|
|
304
|
+
None, alias="Filter", description="Explicitly define the filter for the view."
|
|
305
|
+
)
|
|
306
|
+
in_model: bool = Field(
|
|
307
|
+
True, alias="In Model", description="Indicates whether the view being defined is a part of the data model."
|
|
308
|
+
)
|
|
249
309
|
logical: URIRefType | None = Field(
|
|
250
310
|
None,
|
|
251
311
|
alias="Logical",
|
|
@@ -292,10 +352,14 @@ class DMSView(SheetRow):
|
|
|
292
352
|
|
|
293
353
|
|
|
294
354
|
class DMSNode(SheetRow):
|
|
295
|
-
node: DMSNodeEntity = Field(alias="Node")
|
|
296
|
-
usage: Literal["type", "collection"] = Field(
|
|
297
|
-
|
|
298
|
-
|
|
355
|
+
node: DMSNodeEntity = Field(alias="Node", description="The type definition of the node.")
|
|
356
|
+
usage: Literal["type", "collection"] = Field(
|
|
357
|
+
alias="Usage", description="What the usage of the node is in the data model."
|
|
358
|
+
)
|
|
359
|
+
name: str | None = Field(alias="Name", default=None, description="Human readable name of the node being defined.")
|
|
360
|
+
description: str | None = Field(
|
|
361
|
+
alias="Description", default=None, description="Short description of the node being defined."
|
|
362
|
+
)
|
|
299
363
|
|
|
300
364
|
def _identifier(self) -> tuple[Hashable, ...]:
|
|
301
365
|
return (self.node,)
|
|
@@ -316,10 +380,10 @@ class DMSNode(SheetRow):
|
|
|
316
380
|
|
|
317
381
|
|
|
318
382
|
class DMSEnum(SheetRow):
|
|
319
|
-
collection: ClassEntityType = Field(alias="Collection")
|
|
320
|
-
value: str = Field(alias="Value")
|
|
321
|
-
name: str | None = Field(alias="Name", default=None)
|
|
322
|
-
description: str | None = Field(alias="Description", default=None)
|
|
383
|
+
collection: ClassEntityType = Field(alias="Collection", description="The collection this enum belongs to.")
|
|
384
|
+
value: str = Field(alias="Value", description="The value of the enum.")
|
|
385
|
+
name: str | None = Field(alias="Name", default=None, description="Human readable name of the enum.")
|
|
386
|
+
description: str | None = Field(alias="Description", default=None, description="Short description of the enum.")
|
|
323
387
|
|
|
324
388
|
def _identifier(self) -> tuple[Hashable, ...]:
|
|
325
389
|
return self.collection, self.value
|
|
@@ -332,12 +396,20 @@ class DMSEnum(SheetRow):
|
|
|
332
396
|
|
|
333
397
|
|
|
334
398
|
class DMSRules(BaseRules):
|
|
335
|
-
metadata: DMSMetadata = Field(alias="Metadata")
|
|
336
|
-
properties: SheetList[DMSProperty] = Field(
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
399
|
+
metadata: DMSMetadata = Field(alias="Metadata", description="Contains information about the data model.")
|
|
400
|
+
properties: SheetList[DMSProperty] = Field(
|
|
401
|
+
alias="Properties", description="Contains the properties of the data model."
|
|
402
|
+
)
|
|
403
|
+
views: SheetList[DMSView] = Field(alias="Views", description="Contains the views of the data model.")
|
|
404
|
+
containers: SheetList[DMSContainer] | None = Field(
|
|
405
|
+
None,
|
|
406
|
+
alias="Containers",
|
|
407
|
+
description="Contains the definition containers that are the physical storage of the data model.",
|
|
408
|
+
)
|
|
409
|
+
enum: SheetList[DMSEnum] | None = Field(None, alias="Enum", description="Contains the definition of enum values.")
|
|
410
|
+
nodes: SheetList[DMSNode] | None = Field(
|
|
411
|
+
None, alias="Nodes", description="Contains the definition of the node types."
|
|
412
|
+
)
|
|
341
413
|
|
|
342
414
|
@field_validator("views")
|
|
343
415
|
def matching_version_and_space(cls, value: SheetList[DMSView], info: ValidationInfo) -> SheetList[DMSView]:
|
|
@@ -68,10 +68,16 @@ class InformationClass(SheetRow):
|
|
|
68
68
|
implements: Which classes the current class implements.
|
|
69
69
|
"""
|
|
70
70
|
|
|
71
|
-
class_: ClassEntityType = Field(
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
71
|
+
class_: ClassEntityType = Field(
|
|
72
|
+
alias="Class", description="Class id being defined, use strongly advise `PascalCase` usage."
|
|
73
|
+
)
|
|
74
|
+
name: str | None = Field(alias="Name", default=None, description="Human readable name of the class.")
|
|
75
|
+
description: str | None = Field(alias="Description", default=None, description="Short description of the class.")
|
|
76
|
+
implements: ClassEntityList | None = Field(
|
|
77
|
+
alias="Implements",
|
|
78
|
+
default=None,
|
|
79
|
+
description="List of classes (comma separated) that the current class implements (parents).",
|
|
80
|
+
)
|
|
75
81
|
|
|
76
82
|
physical: URIRefType | None = Field(
|
|
77
83
|
None,
|
|
@@ -119,17 +125,40 @@ class InformationProperty(SheetRow):
|
|
|
119
125
|
knowledge graph. Defaults to None (no transformation)
|
|
120
126
|
"""
|
|
121
127
|
|
|
122
|
-
class_: ClassEntityType = Field(
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
128
|
+
class_: ClassEntityType = Field(
|
|
129
|
+
alias="Class", description="Class id that the property is defined for, strongly advise `PascalCase` usage."
|
|
130
|
+
)
|
|
131
|
+
property_: InformationPropertyType = Field(
|
|
132
|
+
alias="Property", description="Property id, strongly advised to `camelCase` usage."
|
|
133
|
+
)
|
|
134
|
+
name: str | None = Field(alias="Name", default=None, description="Human readable name of the property.")
|
|
135
|
+
description: str | None = Field(alias="Description", default=None, description="Short description of the property.")
|
|
126
136
|
value_type: DataType | ClassEntityType | MultiValueTypeType | UnknownEntity = Field(
|
|
127
|
-
alias="Value Type",
|
|
137
|
+
alias="Value Type",
|
|
138
|
+
union_mode="left_to_right",
|
|
139
|
+
description="Value type that the property can hold. It takes either subset of XSD type or a class defined.",
|
|
140
|
+
)
|
|
141
|
+
min_count: int | None = Field(
|
|
142
|
+
alias="Min Count",
|
|
143
|
+
default=None,
|
|
144
|
+
description="Minimum number of values that the property can hold. "
|
|
145
|
+
"If no value is provided, the default value is `0`, "
|
|
146
|
+
"which means that the property is optional.",
|
|
147
|
+
)
|
|
148
|
+
max_count: int | float | None = Field(
|
|
149
|
+
alias="Max Count",
|
|
150
|
+
default=None,
|
|
151
|
+
description="Maximum number of values that the property can hold. "
|
|
152
|
+
"If no value is provided, the default value is `inf`, "
|
|
153
|
+
"which means that the property can hold any number of values (listable).",
|
|
154
|
+
)
|
|
155
|
+
default: Any | None = Field(alias="Default", default=None, description="Default value of the property.")
|
|
156
|
+
transformation: RDFPath | None = Field(
|
|
157
|
+
alias="Transformation",
|
|
158
|
+
default=None,
|
|
159
|
+
description="The rule that is used to populate the data model. "
|
|
160
|
+
"The rule is provided in a RDFPath query syntax which is converted to downstream solution query (e.g. SPARQL).",
|
|
128
161
|
)
|
|
129
|
-
min_count: int | None = Field(alias="Min Count", default=None)
|
|
130
|
-
max_count: int | float | None = Field(alias="Max Count", default=None)
|
|
131
|
-
default: Any | None = Field(alias="Default", default=None)
|
|
132
|
-
transformation: RDFPath | None = Field(alias="Transformation", default=None)
|
|
133
162
|
inherited: bool = Field(
|
|
134
163
|
default=False,
|
|
135
164
|
exclude=True,
|
|
@@ -221,10 +250,14 @@ class InformationProperty(SheetRow):
|
|
|
221
250
|
|
|
222
251
|
|
|
223
252
|
class InformationRules(BaseRules):
|
|
224
|
-
metadata: InformationMetadata = Field(alias="Metadata")
|
|
225
|
-
properties: SheetList[InformationProperty] = Field(alias="Properties")
|
|
226
|
-
classes: SheetList[InformationClass] = Field(alias="Classes")
|
|
227
|
-
prefixes: dict[str, Namespace] = Field(
|
|
253
|
+
metadata: InformationMetadata = Field(alias="Metadata", description="Metadata for the logical data model")
|
|
254
|
+
properties: SheetList[InformationProperty] = Field(alias="Properties", description="List of properties")
|
|
255
|
+
classes: SheetList[InformationClass] = Field(alias="Classes", description="List of classes")
|
|
256
|
+
prefixes: dict[str, Namespace] = Field(
|
|
257
|
+
alias="Prefixes",
|
|
258
|
+
default_factory=get_default_prefixes,
|
|
259
|
+
description="the definition of the prefixes that are used in the semantic data model",
|
|
260
|
+
)
|
|
228
261
|
|
|
229
262
|
@field_validator("prefixes", mode="before")
|
|
230
263
|
def parse_str(cls, values: Any) -> Any:
|
cognite/neat/_session/_base.py
CHANGED
|
@@ -38,6 +38,24 @@ from .exceptions import NeatSessionError, session_class_wrapper
|
|
|
38
38
|
|
|
39
39
|
@session_class_wrapper
|
|
40
40
|
class NeatSession:
|
|
41
|
+
"""Creates a new NeatSession.
|
|
42
|
+
|
|
43
|
+
This is the main entry point for using Neat. It provides access to the different APIs that can be used to read,
|
|
44
|
+
write, and manipulate data and data models.
|
|
45
|
+
|
|
46
|
+
Args:
|
|
47
|
+
client: The CogniteClient to use for reading and writing data.
|
|
48
|
+
storage: The storage type to use for storing data and data models. Can be either "memory" or "oxigraph".
|
|
49
|
+
In "memory" mode works well for small data sets and when only working with data models. It is works
|
|
50
|
+
well for all notebook environments. In "oxigraph" mode, the data is stored in an Oxigraph database. This
|
|
51
|
+
is more performant for larger data sets and when working with data. Note that this option requires
|
|
52
|
+
additional dependencies to be installed and is not available in CDF Notebooks.
|
|
53
|
+
verbose: Whether to print information about the operations being performed.
|
|
54
|
+
load_engine: Whether to load the Neat Engine. Can be "newest", "cache", or "skip". "newest" will always
|
|
55
|
+
check for the newest version of the engine. "cache" will load the engine if it has been downloaded before.
|
|
56
|
+
"skip" will not load the engine.
|
|
57
|
+
"""
|
|
58
|
+
|
|
41
59
|
def __init__(
|
|
42
60
|
self,
|
|
43
61
|
client: CogniteClient | None = None,
|
cognite/neat/_session/_read.py
CHANGED
|
@@ -17,7 +17,7 @@ from cognite.neat._rules.importers import BaseImporter
|
|
|
17
17
|
from cognite.neat._store._provenance import Activity as ProvenanceActivity
|
|
18
18
|
from cognite.neat._store._provenance import Change
|
|
19
19
|
from cognite.neat._store._provenance import Entity as ProvenanceEntity
|
|
20
|
-
from cognite.neat._utils.reader import GitHubReader, NeatReader, PathReader
|
|
20
|
+
from cognite.neat._utils.reader import GitHubReader, HttpFileReader, NeatReader, PathReader
|
|
21
21
|
|
|
22
22
|
from ._state import SessionState
|
|
23
23
|
from ._wizard import NeatObjectType, RDFFileType, XMLFileType, object_wizard, rdf_dm_wizard, xml_format_wizard
|
|
@@ -244,9 +244,9 @@ class YamlReadAPI(BaseReadAPI):
|
|
|
244
244
|
class CSVReadAPI(BaseReadAPI):
|
|
245
245
|
def __call__(self, io: Any, type: str, primary_key: str) -> None:
|
|
246
246
|
reader = NeatReader.create(io)
|
|
247
|
-
if isinstance(reader,
|
|
247
|
+
if isinstance(reader, HttpFileReader):
|
|
248
248
|
path = Path(tempfile.gettempdir()).resolve() / reader.name
|
|
249
|
-
path.write_text(reader.read_text())
|
|
249
|
+
path.write_text(reader.read_text(), encoding="utf-8", newline="\n")
|
|
250
250
|
elif isinstance(reader, PathReader):
|
|
251
251
|
path = reader.path
|
|
252
252
|
else:
|
cognite/neat/_session/_to.py
CHANGED
|
@@ -132,7 +132,7 @@ class CDFToAPI:
|
|
|
132
132
|
dry_run: bool = False,
|
|
133
133
|
drop_data: bool = False,
|
|
134
134
|
components: Component | Collection[Component] | None = None,
|
|
135
|
-
):
|
|
135
|
+
) -> UploadResultList:
|
|
136
136
|
"""Export the verified DMS data model to CDF.
|
|
137
137
|
|
|
138
138
|
Args:
|
|
@@ -150,7 +150,7 @@ class CDFToAPI:
|
|
|
150
150
|
- "skip": If any component already exists, it will be skipped.
|
|
151
151
|
- "update": If any component already exists, it will be updated.
|
|
152
152
|
- "force": If any component already exists, and the update fails, it will be deleted and recreated.
|
|
153
|
-
- "recreate": All components will be deleted and recreated.
|
|
153
|
+
- "recreate": All components will be deleted and recreated. The exception is spaces, which will be updated.
|
|
154
154
|
|
|
155
155
|
"""
|
|
156
156
|
|
|
@@ -164,5 +164,5 @@ class CDFToAPI:
|
|
|
164
164
|
result = exporter.export_to_cdf(self._state.data_model.last_verified_dms_rules[1], self._client, dry_run)
|
|
165
165
|
result.insert(0, UploadResultCore(name="schema", issues=conversion_issues))
|
|
166
166
|
self._state.data_model.outcome.append(result)
|
|
167
|
-
print("You can inspect the details with the .inspect.data_model
|
|
167
|
+
print("You can inspect the details with the .inspect.outcome.data_model(...) method.")
|
|
168
168
|
return result
|
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
from ._base import GitHubReader, NeatReader, PathReader
|
|
1
|
+
from ._base import GitHubReader, HttpFileReader, NeatReader, PathReader
|
|
2
2
|
|
|
3
|
-
__all__ = ["NeatReader", "PathReader", "GitHubReader"]
|
|
3
|
+
__all__ = ["NeatReader", "PathReader", "GitHubReader", "HttpFileReader"]
|
|
@@ -17,6 +17,8 @@ class NeatReader(ABC):
|
|
|
17
17
|
url = urlparse(io)
|
|
18
18
|
if url.scheme == "https" and url.netloc.endswith("github.com"):
|
|
19
19
|
return GitHubReader(io)
|
|
20
|
+
elif url.scheme == "https":
|
|
21
|
+
return HttpFileReader(io, url.path)
|
|
20
22
|
|
|
21
23
|
if isinstance(io, str | Path):
|
|
22
24
|
return PathReader(Path(io))
|
|
@@ -94,12 +96,10 @@ class PathReader(NeatReader):
|
|
|
94
96
|
return self.path.exists()
|
|
95
97
|
|
|
96
98
|
|
|
97
|
-
class
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
self.raw = raw
|
|
102
|
-
self.repo, self.path = self._parse_url(raw)
|
|
99
|
+
class HttpFileReader(NeatReader):
|
|
100
|
+
def __init__(self, url: str, path: str):
|
|
101
|
+
self._url = url
|
|
102
|
+
self.path = path
|
|
103
103
|
|
|
104
104
|
@property
|
|
105
105
|
def name(self) -> str:
|
|
@@ -107,9 +107,40 @@ class GitHubReader(NeatReader):
|
|
|
107
107
|
return self.path.rsplit("/", maxsplit=1)[-1]
|
|
108
108
|
return self.path
|
|
109
109
|
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
110
|
+
def read_text(self) -> str:
|
|
111
|
+
response = requests.get(self._url)
|
|
112
|
+
response.raise_for_status()
|
|
113
|
+
return response.text
|
|
114
|
+
|
|
115
|
+
def size(self) -> int:
|
|
116
|
+
response = requests.head(self._url)
|
|
117
|
+
response.raise_for_status()
|
|
118
|
+
return int(response.headers["Content-Length"])
|
|
119
|
+
|
|
120
|
+
def iterate(self, chunk_size: int) -> Iterable[str]:
|
|
121
|
+
with requests.get(self._url, stream=True) as response:
|
|
122
|
+
response.raise_for_status()
|
|
123
|
+
for chunk in response.iter_content(chunk_size):
|
|
124
|
+
yield chunk.decode("utf-8")
|
|
125
|
+
|
|
126
|
+
def __enter__(self) -> IO:
|
|
127
|
+
return StringIO(self.read_text())
|
|
128
|
+
|
|
129
|
+
def __str__(self) -> str:
|
|
130
|
+
return self._url
|
|
131
|
+
|
|
132
|
+
def exists(self) -> bool:
|
|
133
|
+
response = requests.head(self._url)
|
|
134
|
+
return 200 <= response.status_code < 400
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
class GitHubReader(HttpFileReader):
|
|
138
|
+
raw_url = "https://raw.githubusercontent.com/"
|
|
139
|
+
|
|
140
|
+
def __init__(self, raw: str):
|
|
141
|
+
self.raw = raw
|
|
142
|
+
self.repo, path = self._parse_url(raw)
|
|
143
|
+
super().__init__(f"{self.raw_url}{self.repo}/main/{path}", path)
|
|
113
144
|
|
|
114
145
|
@staticmethod
|
|
115
146
|
def _parse_url(url: str) -> tuple[str, str]:
|
|
@@ -134,29 +165,3 @@ class GitHubReader(NeatReader):
|
|
|
134
165
|
|
|
135
166
|
def __str__(self) -> str:
|
|
136
167
|
return self.raw
|
|
137
|
-
|
|
138
|
-
def read_text(self) -> str:
|
|
139
|
-
response = requests.get(self._full_url)
|
|
140
|
-
response.raise_for_status()
|
|
141
|
-
return response.text
|
|
142
|
-
|
|
143
|
-
def size(self) -> int:
|
|
144
|
-
response = requests.head(self._full_url)
|
|
145
|
-
response.raise_for_status()
|
|
146
|
-
return int(response.headers["Content-Length"])
|
|
147
|
-
|
|
148
|
-
def iterate(self, chunk_size: int) -> Iterable[str]:
|
|
149
|
-
with requests.get(self._full_url, stream=True) as response:
|
|
150
|
-
response.raise_for_status()
|
|
151
|
-
for chunk in response.iter_content(chunk_size):
|
|
152
|
-
yield chunk.decode("utf-8")
|
|
153
|
-
|
|
154
|
-
def __enter__(self) -> IO:
|
|
155
|
-
return StringIO(self.read_text())
|
|
156
|
-
|
|
157
|
-
def __exit__(self, exc_type, exc_value, traceback) -> None:
|
|
158
|
-
pass
|
|
159
|
-
|
|
160
|
-
def exists(self) -> bool:
|
|
161
|
-
response = requests.head(self._full_url)
|
|
162
|
-
return 200 <= response.status_code < 400
|
cognite/neat/_utils/text.py
CHANGED
|
@@ -106,6 +106,18 @@ def to_snake(string: str) -> str:
|
|
|
106
106
|
return "_".join(map(str.lower, words))
|
|
107
107
|
|
|
108
108
|
|
|
109
|
+
def sentence_or_string_to_camel(string: str) -> str:
|
|
110
|
+
# Could be a combination of kebab and pascal/camel case
|
|
111
|
+
if " " in string:
|
|
112
|
+
parts = string.split(" ")
|
|
113
|
+
try:
|
|
114
|
+
return parts[0].casefold() + "".join(word.capitalize() for word in parts[1:])
|
|
115
|
+
except IndexError:
|
|
116
|
+
return ""
|
|
117
|
+
else:
|
|
118
|
+
return to_camel(string)
|
|
119
|
+
|
|
120
|
+
|
|
109
121
|
def replace_non_alphanumeric_with_underscore(text: str) -> str:
|
|
110
122
|
return re.sub(r"\W+", "_", text)
|
|
111
123
|
|
cognite/neat/_version.py
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
__version__ = "0.
|
|
1
|
+
__version__ = "0.101.0"
|
|
2
2
|
__engine__ = "^2.0.1"
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: cognite-neat
|
|
3
|
+
Version: 0.101.0
|
|
4
|
+
Summary: Knowledge graph transformation
|
|
5
|
+
Home-page: https://cognite-neat.readthedocs-hosted.com/
|
|
6
|
+
License: Apache-2.0
|
|
7
|
+
Author: Nikola Vasiljevic
|
|
8
|
+
Author-email: nikola.vasiljevic@cognite.com
|
|
9
|
+
Requires-Python: >=3.10,<4.0
|
|
10
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
16
|
+
Provides-Extra: all
|
|
17
|
+
Provides-Extra: docs
|
|
18
|
+
Provides-Extra: google
|
|
19
|
+
Provides-Extra: oxi
|
|
20
|
+
Provides-Extra: service
|
|
21
|
+
Requires-Dist: PyYAML
|
|
22
|
+
Requires-Dist: backports.strenum (>=1.2,<2.0) ; python_version < "3.11"
|
|
23
|
+
Requires-Dist: cognite-sdk (>=7.54.6,<8.0.0)
|
|
24
|
+
Requires-Dist: elementpath (>=4.0.0,<5.0.0)
|
|
25
|
+
Requires-Dist: exceptiongroup (>=1.1.3,<2.0.0) ; python_version < "3.11"
|
|
26
|
+
Requires-Dist: fastapi (>=0,<1) ; extra == "service" or extra == "all"
|
|
27
|
+
Requires-Dist: google-api-python-client ; extra == "google"
|
|
28
|
+
Requires-Dist: google-auth-oauthlib ; extra == "google"
|
|
29
|
+
Requires-Dist: gspread ; extra == "google"
|
|
30
|
+
Requires-Dist: jsonpath-python (>=1.0.6,<2.0.0)
|
|
31
|
+
Requires-Dist: lxml (>=5.3.0,<6.0.0) ; extra == "all"
|
|
32
|
+
Requires-Dist: mixpanel (>=4.10.1,<5.0.0)
|
|
33
|
+
Requires-Dist: mkdocs ; extra == "docs"
|
|
34
|
+
Requires-Dist: mkdocs-autorefs (>=0.5.0,<0.6.0) ; extra == "docs"
|
|
35
|
+
Requires-Dist: mkdocs-git-authors-plugin ; extra == "docs"
|
|
36
|
+
Requires-Dist: mkdocs-git-revision-date-localized-plugin ; extra == "docs"
|
|
37
|
+
Requires-Dist: mkdocs-gitbook ; extra == "docs"
|
|
38
|
+
Requires-Dist: mkdocs-glightbox ; extra == "docs"
|
|
39
|
+
Requires-Dist: mkdocs-jupyter ; extra == "docs"
|
|
40
|
+
Requires-Dist: mkdocs-material-extensions ; extra == "docs"
|
|
41
|
+
Requires-Dist: mkdocstrings[python] ; extra == "docs"
|
|
42
|
+
Requires-Dist: networkx (>=3.4.2,<4.0.0)
|
|
43
|
+
Requires-Dist: openpyxl
|
|
44
|
+
Requires-Dist: oxrdflib[oxigraph] (>=0.4.0,<0.5.0) ; extra == "oxi" or extra == "all"
|
|
45
|
+
Requires-Dist: packaging (>=22.0,<25.0)
|
|
46
|
+
Requires-Dist: pandas
|
|
47
|
+
Requires-Dist: prometheus-client (>=0,<1) ; extra == "service" or extra == "all"
|
|
48
|
+
Requires-Dist: pydantic (>=2,<3)
|
|
49
|
+
Requires-Dist: pymdown-extensions ; extra == "docs"
|
|
50
|
+
Requires-Dist: pyoxigraph (==0.4.3) ; extra == "oxi" or extra == "all"
|
|
51
|
+
Requires-Dist: python-multipart (==0.0.9) ; extra == "service" or extra == "all"
|
|
52
|
+
Requires-Dist: pyvis (>=0.3.2,<0.4.0)
|
|
53
|
+
Requires-Dist: rdflib
|
|
54
|
+
Requires-Dist: requests
|
|
55
|
+
Requires-Dist: rich[jupyter] (>=13.7.1,<14.0.0)
|
|
56
|
+
Requires-Dist: schedule (>=1,<2) ; extra == "service" or extra == "all"
|
|
57
|
+
Requires-Dist: tomli (>=2.0.1,<3.0.0) ; python_version < "3.11"
|
|
58
|
+
Requires-Dist: typing_extensions (>=4.8,<5.0) ; python_version < "3.11"
|
|
59
|
+
Requires-Dist: urllib3 (>=2,<3)
|
|
60
|
+
Requires-Dist: uvicorn[standard] (>=0,<1) ; extra == "service" or extra == "all"
|
|
61
|
+
Project-URL: Documentation, https://cognite-neat.readthedocs-hosted.com/
|
|
62
|
+
Project-URL: Repository, https://github.com/cognitedata/neat
|
|
63
|
+
Description-Content-Type: text/markdown
|
|
64
|
+
|
|
65
|
+
# kNowlEdge grAph Transformer (NEAT)
|
|
66
|
+
|
|
67
|
+
[](https://github.com/cognitedata/neat/actions/workflows/release.yaml)
|
|
68
|
+
[](https://cognite-neat.readthedocs-hosted.com/en/latest/?badge=latest)
|
|
69
|
+
[](https://github.com/cognitedata/neat)
|
|
70
|
+
[](https://pypi.org/project/cognite-neat/)
|
|
71
|
+
[](https://pypistats.org/packages/cognite-neat)
|
|
72
|
+
[](https://hub.docker.com/r/cognite/neat)
|
|
73
|
+
[](https://github.com/cognitedata/neat/blob/master/LICENSE)
|
|
74
|
+
[](https://github.com/ambv/black)
|
|
75
|
+
[](https://github.com/astral-sh/ruff)
|
|
76
|
+
[](http://mypy-lang.org)
|
|
77
|
+
|
|
78
|
+
NEAT is a domain expert centric and developer friendly solution for rapid:
|
|
79
|
+
|
|
80
|
+
- semantic data modeling
|
|
81
|
+
- creation, transformation and enrichment of knowledge graphs
|
|
82
|
+
- and ingestion of the models and graphs into [Cognite Data Fusion](https://www.cognite.com/en/product/cognite_data_fusion_industrial_dataops_platform)
|
|
83
|
+
|
|
84
|
+
NEAT is using open and globally recognized standards maintained by the [World Wide Web Consortium (W3C)](https://www.w3.org/RDF/).
|
|
85
|
+
NEAT represents an essential tool for creation of standardized, machine-actionable, linked and semantic (meta)data.
|
|
86
|
+
|
|
87
|
+
> NEAT is an acronym derived from k**N**owl**Ed**ge gr**A**ph **T**ransformer hallucinated by GenAI.
|
|
88
|
+
|
|
89
|
+
## Installation
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
pip install cognite-neat
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
## Usage
|
|
96
|
+
|
|
97
|
+
The user interface for `NEAT` is a notebook-based environment. Once you have set up your notebook
|
|
98
|
+
environment, you start by creating a `CogniteClient` and instantiate a `NeatSession` object.
|
|
99
|
+
|
|
100
|
+
```python
|
|
101
|
+
from cognite.neat import NeatSession, get_cognite_client
|
|
102
|
+
|
|
103
|
+
client = get_cognite_client(".env")
|
|
104
|
+
|
|
105
|
+
neat = NeatSession(client)
|
|
106
|
+
|
|
107
|
+
neat.read.cdf.data_model(("my_space", "MyDataModel", "v1"))
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
## Documentation
|
|
111
|
+
|
|
112
|
+
For more information, see the [documentation](https://cognite-neat.readthedocs-hosted.com/en/latest/)
|
|
113
|
+
|