cognite-neat 0.92.3__py3-none-any.whl → 0.94.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/__init__.py +3 -2
- cognite/neat/{app → _app}/api/configuration.py +9 -7
- cognite/neat/_app/api/context_manager/__init__.py +3 -0
- cognite/neat/{app → _app}/api/context_manager/manager.py +1 -1
- cognite/neat/{app → _app}/api/explorer.py +5 -5
- cognite/neat/{app → _app}/api/routers/configuration.py +2 -2
- cognite/neat/{app → _app}/api/routers/crud.py +4 -4
- cognite/neat/{app → _app}/api/routers/workflows.py +7 -7
- cognite/neat/{app → _app}/main.py +1 -1
- cognite/neat/_app/ui/neat-app/package-lock.json +18306 -0
- cognite/neat/_app/ui/neat-app/package.json +62 -0
- cognite/neat/_app/ui/neat-app/public/favicon.ico +0 -0
- cognite/neat/_app/ui/neat-app/public/img/architect-icon.svg +116 -0
- cognite/neat/_app/ui/neat-app/public/img/developer-icon.svg +112 -0
- cognite/neat/_app/ui/neat-app/public/img/sme-icon.svg +34 -0
- cognite/neat/_app/ui/neat-app/public/index.html +43 -0
- cognite/neat/_app/ui/neat-app/public/logo192.png +0 -0
- cognite/neat/_app/ui/neat-app/public/manifest.json +25 -0
- cognite/neat/_app/ui/neat-app/public/robots.txt +3 -0
- cognite/neat/_app/ui/neat-app/src/App.css +38 -0
- cognite/neat/_app/ui/neat-app/src/App.js +17 -0
- cognite/neat/_app/ui/neat-app/src/App.test.js +8 -0
- cognite/neat/_app/ui/neat-app/src/MainContainer.tsx +70 -0
- cognite/neat/_app/ui/neat-app/src/components/JsonViewer.tsx +43 -0
- cognite/neat/_app/ui/neat-app/src/components/LocalUploader.tsx +124 -0
- cognite/neat/_app/ui/neat-app/src/components/OverviewComponentEditorDialog.tsx +63 -0
- cognite/neat/_app/ui/neat-app/src/components/StepEditorDialog.tsx +511 -0
- cognite/neat/_app/ui/neat-app/src/components/TabPanel.tsx +36 -0
- cognite/neat/_app/ui/neat-app/src/components/Utils.tsx +56 -0
- cognite/neat/_app/ui/neat-app/src/components/WorkflowDeleteDialog.tsx +60 -0
- cognite/neat/_app/ui/neat-app/src/components/WorkflowExecutionReport.tsx +112 -0
- cognite/neat/_app/ui/neat-app/src/components/WorkflowImportExportDialog.tsx +67 -0
- cognite/neat/_app/ui/neat-app/src/components/WorkflowMetadataDialog.tsx +79 -0
- cognite/neat/_app/ui/neat-app/src/index.css +13 -0
- cognite/neat/_app/ui/neat-app/src/index.js +13 -0
- cognite/neat/_app/ui/neat-app/src/logo.svg +1 -0
- cognite/neat/_app/ui/neat-app/src/reportWebVitals.js +13 -0
- cognite/neat/_app/ui/neat-app/src/setupTests.js +5 -0
- cognite/neat/_app/ui/neat-app/src/types/WorkflowTypes.ts +388 -0
- cognite/neat/_app/ui/neat-app/src/views/AboutView.tsx +61 -0
- cognite/neat/_app/ui/neat-app/src/views/ConfigView.tsx +184 -0
- cognite/neat/_app/ui/neat-app/src/views/GlobalConfigView.tsx +180 -0
- cognite/neat/_app/ui/neat-app/src/views/WorkflowView.tsx +570 -0
- cognite/neat/_app/ui/neat-app/tsconfig.json +27 -0
- cognite/neat/{config.py → _config.py} +3 -3
- cognite/neat/{constants.py → _constants.py} +13 -5
- cognite/neat/_graph/_shared.py +34 -0
- cognite/neat/{graph → _graph}/_tracking/base.py +1 -1
- cognite/neat/{graph → _graph}/_tracking/log.py +1 -1
- cognite/neat/{graph → _graph}/extractors/__init__.py +5 -0
- cognite/neat/{graph → _graph}/extractors/_base.py +2 -2
- cognite/neat/{graph → _graph}/extractors/_classic_cdf/_assets.py +1 -1
- cognite/neat/{graph → _graph}/extractors/_classic_cdf/_base.py +4 -4
- cognite/neat/{graph → _graph}/extractors/_classic_cdf/_classic.py +5 -5
- cognite/neat/{graph → _graph}/extractors/_classic_cdf/_data_sets.py +1 -1
- cognite/neat/{graph → _graph}/extractors/_classic_cdf/_events.py +1 -1
- cognite/neat/{graph → _graph}/extractors/_classic_cdf/_files.py +1 -1
- cognite/neat/{graph → _graph}/extractors/_classic_cdf/_labels.py +1 -1
- cognite/neat/{graph → _graph}/extractors/_classic_cdf/_relationships.py +2 -2
- cognite/neat/{graph → _graph}/extractors/_classic_cdf/_sequences.py +1 -1
- cognite/neat/{graph → _graph}/extractors/_classic_cdf/_timeseries.py +1 -1
- cognite/neat/{graph → _graph}/extractors/_dexpi.py +5 -5
- cognite/neat/{graph → _graph}/extractors/_dms.py +3 -3
- cognite/neat/_graph/extractors/_iodd.py +402 -0
- cognite/neat/{graph → _graph}/extractors/_mock_graph_generator.py +9 -8
- cognite/neat/_graph/extractors/_rdf_file.py +49 -0
- cognite/neat/{graph → _graph}/loaders/_base.py +5 -5
- cognite/neat/{graph → _graph}/loaders/_rdf2asset.py +11 -10
- cognite/neat/{graph → _graph}/loaders/_rdf2dms.py +10 -10
- cognite/neat/{graph → _graph}/queries/_base.py +91 -19
- cognite/neat/{graph → _graph}/queries/_construct.py +5 -5
- cognite/neat/{graph → _graph}/queries/_shared.py +3 -3
- cognite/neat/{graph → _graph}/transformers/__init__.py +6 -0
- cognite/neat/{graph → _graph}/transformers/_classic_cdf.py +135 -3
- cognite/neat/_graph/transformers/_iodd.py +25 -0
- cognite/neat/_graph/transformers/_prune_graph.py +126 -0
- cognite/neat/{graph → _graph}/transformers/_rdfpath.py +3 -3
- cognite/neat/_graph/transformers/_value_type.py +66 -0
- cognite/neat/{issues → _issues}/_base.py +32 -17
- cognite/neat/{issues → _issues}/errors/__init__.py +1 -1
- cognite/neat/{issues → _issues}/errors/_external.py +8 -8
- cognite/neat/{issues → _issues}/errors/_general.py +5 -5
- cognite/neat/{issues → _issues}/errors/_properties.py +7 -7
- cognite/neat/{issues → _issues}/errors/_resources.py +11 -11
- cognite/neat/{issues → _issues}/errors/_workflow.py +5 -5
- cognite/neat/{issues → _issues}/warnings/__init__.py +1 -1
- cognite/neat/{issues → _issues}/warnings/_external.py +5 -5
- cognite/neat/{issues → _issues}/warnings/_general.py +4 -4
- cognite/neat/{issues → _issues}/warnings/_models.py +10 -10
- cognite/neat/{issues → _issues}/warnings/_properties.py +6 -6
- cognite/neat/{issues → _issues}/warnings/_resources.py +5 -5
- cognite/neat/{issues → _issues}/warnings/user_modeling.py +9 -9
- cognite/neat/_rules/_constants.py +190 -0
- cognite/neat/{rules → _rules}/_shared.py +5 -5
- cognite/neat/_rules/analysis/__init__.py +5 -0
- cognite/neat/{rules → _rules}/analysis/_asset.py +5 -5
- cognite/neat/{rules → _rules}/analysis/_base.py +5 -5
- cognite/neat/_rules/analysis/_dms.py +43 -0
- cognite/neat/{rules → _rules}/analysis/_information.py +12 -6
- cognite/neat/_rules/catalog/__init__.py +6 -0
- cognite/neat/_rules/catalog/info-rules-imf.xlsx +0 -0
- cognite/neat/{rules → _rules}/exporters/__init__.py +2 -0
- cognite/neat/{rules → _rules}/exporters/_base.py +3 -3
- cognite/neat/{rules → _rules}/exporters/_rules2dms.py +5 -5
- cognite/neat/{rules → _rules}/exporters/_rules2excel.py +12 -8
- cognite/neat/_rules/exporters/_rules2instance_template.py +152 -0
- cognite/neat/{rules → _rules}/exporters/_rules2ontology.py +10 -9
- cognite/neat/{rules → _rules}/exporters/_rules2yaml.py +1 -3
- cognite/neat/{rules → _rules}/exporters/_validation.py +2 -2
- cognite/neat/{rules → _rules}/importers/_base.py +3 -3
- cognite/neat/{rules → _rules}/importers/_dms2rules.py +9 -9
- cognite/neat/{rules → _rules}/importers/_dtdl2rules/dtdl_converter.py +7 -7
- cognite/neat/{rules → _rules}/importers/_dtdl2rules/dtdl_importer.py +9 -9
- cognite/neat/{rules → _rules}/importers/_dtdl2rules/spec.py +1 -1
- cognite/neat/_rules/importers/_rdf/_base.py +144 -0
- cognite/neat/{rules → _rules}/importers/_rdf/_imf2rules/_imf2classes.py +1 -1
- cognite/neat/{rules → _rules}/importers/_rdf/_imf2rules/_imf2metadata.py +4 -4
- cognite/neat/{rules → _rules}/importers/_rdf/_imf2rules/_imf2properties.py +2 -1
- cognite/neat/{rules → _rules}/importers/_rdf/_imf2rules/_imf2rules.py +8 -39
- cognite/neat/{rules → _rules}/importers/_rdf/_inference2rules.py +33 -106
- cognite/neat/{rules → _rules}/importers/_rdf/_owl2rules/_owl2classes.py +1 -1
- cognite/neat/{rules → _rules}/importers/_rdf/_owl2rules/_owl2metadata.py +5 -5
- cognite/neat/{rules → _rules}/importers/_rdf/_owl2rules/_owl2properties.py +1 -1
- cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2rules.py +39 -0
- cognite/neat/{rules → _rules}/importers/_rdf/_shared.py +4 -4
- cognite/neat/{rules → _rules}/importers/_spreadsheet2rules.py +7 -7
- cognite/neat/{rules → _rules}/importers/_yaml2rules.py +5 -5
- cognite/neat/{rules → _rules}/models/__init__.py +5 -5
- cognite/neat/{rules → _rules}/models/_base_input.py +15 -6
- cognite/neat/{rules → _rules}/models/_base_rules.py +14 -2
- cognite/neat/{rules → _rules}/models/_rdfpath.py +1 -1
- cognite/neat/_rules/models/_types.py +151 -0
- cognite/neat/{rules → _rules}/models/asset/_rules.py +4 -4
- cognite/neat/{rules → _rules}/models/asset/_rules_input.py +4 -4
- cognite/neat/{rules → _rules}/models/asset/_validation.py +7 -7
- cognite/neat/{rules → _rules}/models/data_types.py +15 -12
- cognite/neat/{rules → _rules}/models/dms/_exporter.py +60 -12
- cognite/neat/{rules → _rules}/models/dms/_rules.py +26 -23
- cognite/neat/{rules → _rules}/models/dms/_rules_input.py +4 -4
- cognite/neat/{rules → _rules}/models/dms/_schema.py +15 -14
- cognite/neat/{rules → _rules}/models/dms/_validation.py +8 -8
- cognite/neat/{rules → _rules}/models/domain.py +6 -6
- cognite/neat/{rules → _rules}/models/entities/__init__.py +1 -2
- cognite/neat/_rules/models/entities/_constants.py +15 -0
- cognite/neat/{rules → _rules}/models/entities/_loaders.py +2 -2
- cognite/neat/{rules → _rules}/models/entities/_multi_value.py +15 -2
- cognite/neat/{rules → _rules}/models/entities/_single_value.py +7 -4
- cognite/neat/{rules → _rules}/models/information/_rules.py +34 -22
- cognite/neat/{rules → _rules}/models/information/_rules_input.py +3 -3
- cognite/neat/{rules → _rules}/models/information/_validation.py +6 -5
- cognite/neat/_rules/models/mapping/__init__.py +4 -0
- cognite/neat/_rules/models/mapping/_base.py +131 -0
- cognite/neat/_rules/models/mapping/_classic2core.py +150 -0
- cognite/neat/{rules → _rules}/transformers/__init__.py +15 -2
- cognite/neat/{rules → _rules}/transformers/_base.py +3 -3
- cognite/neat/{rules → _rules}/transformers/_converters.py +289 -20
- cognite/neat/{rules/transformers/_map_onto.py → _rules/transformers/_mapping.py} +46 -4
- cognite/neat/{rules → _rules}/transformers/_pipelines.py +4 -4
- cognite/neat/{rules → _rules}/transformers/_verification.py +10 -4
- cognite/neat/_session/__init__.py +3 -0
- cognite/neat/_session/_base.py +86 -0
- cognite/neat/_session/_prepare.py +61 -0
- cognite/neat/_session/_read.py +118 -0
- cognite/neat/_session/_show.py +274 -0
- cognite/neat/_session/_state.py +69 -0
- cognite/neat/_session/_to.py +70 -0
- cognite/neat/_session/_wizard.py +39 -0
- cognite/neat/_session/exceptions.py +42 -0
- cognite/neat/{store → _store}/_base.py +62 -31
- cognite/neat/{store → _store}/_provenance.py +11 -1
- cognite/neat/{utils → _utils}/auth.py +14 -3
- cognite/neat/{utils → _utils}/auxiliary.py +1 -1
- cognite/neat/{utils → _utils}/cdf/loaders/_data_modeling.py +8 -2
- cognite/neat/{utils → _utils}/cdf/loaders/_ingestion.py +1 -1
- cognite/neat/{utils → _utils}/upload.py +1 -1
- cognite/neat/_version.py +1 -1
- cognite/neat/_workflows/__init__.py +17 -0
- cognite/neat/{workflows → _workflows}/base.py +10 -10
- cognite/neat/{workflows → _workflows}/cdf_store.py +3 -3
- cognite/neat/{workflows → _workflows}/examples/Export_DMS/workflow.yaml +89 -89
- cognite/neat/{workflows → _workflows}/manager.py +6 -6
- cognite/neat/{workflows → _workflows}/steps/data_contracts.py +3 -3
- cognite/neat/{workflows → _workflows}/steps/lib/current/graph_extractor.py +8 -31
- cognite/neat/{workflows → _workflows}/steps/lib/current/graph_loader.py +4 -4
- cognite/neat/{workflows → _workflows}/steps/lib/current/graph_store.py +4 -4
- cognite/neat/{workflows → _workflows}/steps/lib/current/rules_exporter.py +8 -8
- cognite/neat/{workflows → _workflows}/steps/lib/current/rules_importer.py +13 -13
- cognite/neat/{workflows → _workflows}/steps/lib/current/rules_validator.py +8 -8
- cognite/neat/{workflows → _workflows}/steps/lib/io/io_steps.py +3 -3
- cognite/neat/{workflows → _workflows}/steps/step_model.py +3 -3
- cognite/neat/{workflows → _workflows}/steps_registry.py +9 -9
- cognite/neat/{workflows → _workflows}/tasks.py +1 -1
- cognite/neat/{workflows → _workflows}/triggers.py +2 -2
- {cognite_neat-0.92.3.dist-info → cognite_neat-0.94.0.dist-info}/METADATA +6 -2
- cognite_neat-0.94.0.dist-info/RECORD +276 -0
- {cognite_neat-0.92.3.dist-info → cognite_neat-0.94.0.dist-info}/WHEEL +1 -1
- cognite_neat-0.94.0.dist-info/entry_points.txt +3 -0
- cognite/neat/app/api/context_manager/__init__.py +0 -3
- cognite/neat/graph/_shared.py +0 -5
- cognite/neat/graph/extractors/_iodd.py +0 -160
- cognite/neat/graph/extractors/_rdf_file.py +0 -26
- cognite/neat/rules/analysis/__init__.py +0 -6
- cognite/neat/rules/examples/__init__.py +0 -10
- cognite/neat/rules/examples/info-rules-imf.xlsx +0 -0
- cognite/neat/rules/examples/wind-energy.owl +0 -1511
- cognite/neat/rules/importers/_rdf/_owl2rules/_owl2rules.py +0 -65
- cognite/neat/rules/models/_types.py +0 -96
- cognite/neat/rules/models/entities/_constants.py +0 -73
- cognite/neat/utils/regex_patterns.py +0 -58
- cognite/neat/workflows/__init__.py +0 -12
- cognite_neat-0.92.3.dist-info/RECORD +0 -224
- cognite_neat-0.92.3.dist-info/entry_points.txt +0 -3
- /cognite/neat/{app → _app}/api/__init__.py +0 -0
- /cognite/neat/{app → _app}/api/asgi/metrics.py +0 -0
- /cognite/neat/{app → _app}/api/data_classes/__init__.py +0 -0
- /cognite/neat/{app → _app}/api/data_classes/rest.py +0 -0
- /cognite/neat/{app → _app}/api/routers/metrics.py +0 -0
- /cognite/neat/{app → _app}/api/utils/__init__.py +0 -0
- /cognite/neat/{app → _app}/api/utils/data_mapping.py +0 -0
- /cognite/neat/{app → _app}/api/utils/logging.py +0 -0
- /cognite/neat/{app → _app}/api/utils/query_templates.py +0 -0
- /cognite/neat/{app → _app}/monitoring/__init__.py +0 -0
- /cognite/neat/{app → _app}/monitoring/metrics.py +0 -0
- /cognite/neat/{app → _app}/ui/index.html +0 -0
- /cognite/neat/{app → _app}/ui/neat-app/.gitignore +0 -0
- /cognite/neat/{app → _app}/ui/neat-app/README.md +0 -0
- /cognite/neat/{app → _app}/ui/neat-app/build/asset-manifest.json +0 -0
- /cognite/neat/{app → _app}/ui/neat-app/build/favicon.ico +0 -0
- /cognite/neat/{app → _app}/ui/neat-app/build/img/architect-icon.svg +0 -0
- /cognite/neat/{app → _app}/ui/neat-app/build/img/developer-icon.svg +0 -0
- /cognite/neat/{app → _app}/ui/neat-app/build/img/sme-icon.svg +0 -0
- /cognite/neat/{app → _app}/ui/neat-app/build/index.html +0 -0
- /cognite/neat/{app → _app}/ui/neat-app/build/logo192.png +0 -0
- /cognite/neat/{app → _app}/ui/neat-app/build/manifest.json +0 -0
- /cognite/neat/{app → _app}/ui/neat-app/build/robots.txt +0 -0
- /cognite/neat/{app → _app}/ui/neat-app/build/static/css/main.72e3d92e.css +0 -0
- /cognite/neat/{app → _app}/ui/neat-app/build/static/css/main.72e3d92e.css.map +0 -0
- /cognite/neat/{app → _app}/ui/neat-app/build/static/js/main.5a52cf09.js +0 -0
- /cognite/neat/{app → _app}/ui/neat-app/build/static/js/main.5a52cf09.js.LICENSE.txt +0 -0
- /cognite/neat/{app → _app}/ui/neat-app/build/static/js/main.5a52cf09.js.map +0 -0
- /cognite/neat/{app → _app}/ui/neat-app/build/static/media/logo.8093b84df9ed36a174c629d6fe0b730d.svg +0 -0
- /cognite/neat/{graph → _graph}/__init__.py +0 -0
- /cognite/neat/{graph → _graph}/_tracking/__init__.py +0 -0
- /cognite/neat/{graph → _graph}/examples/Knowledge-Graph-Nordic44-dirty.xml +0 -0
- /cognite/neat/{graph → _graph}/examples/Knowledge-Graph-Nordic44.xml +0 -0
- /cognite/neat/{graph → _graph}/examples/__init__.py +0 -0
- /cognite/neat/{graph → _graph}/examples/skos-capturing-sheet-wind-topics.xlsx +0 -0
- /cognite/neat/{graph → _graph}/extractors/_classic_cdf/__init__.py +0 -0
- /cognite/neat/{graph → _graph}/loaders/__init__.py +0 -0
- /cognite/neat/{graph → _graph}/models.py +0 -0
- /cognite/neat/{graph → _graph}/queries/__init__.py +0 -0
- /cognite/neat/{graph → _graph}/transformers/_base.py +0 -0
- /cognite/neat/{issues → _issues}/__init__.py +0 -0
- /cognite/neat/{issues → _issues}/formatters.py +0 -0
- /cognite/neat/{rules → _rules}/__init__.py +0 -0
- /cognite/neat/{rules → _rules}/importers/__init__.py +0 -0
- /cognite/neat/{rules → _rules}/importers/_dtdl2rules/__init__.py +0 -0
- /cognite/neat/{rules → _rules}/importers/_dtdl2rules/_unit_lookup.py +0 -0
- /cognite/neat/{rules → _rules}/importers/_rdf/__init__.py +0 -0
- /cognite/neat/{rules → _rules}/importers/_rdf/_imf2rules/__init__.py +0 -0
- /cognite/neat/{rules → _rules}/importers/_rdf/_owl2rules/__init__.py +0 -0
- /cognite/neat/{rules → _rules}/models/asset/__init__.py +0 -0
- /cognite/neat/{rules → _rules}/models/dms/__init__.py +0 -0
- /cognite/neat/{rules → _rules}/models/entities/_types.py +0 -0
- /cognite/neat/{rules → _rules}/models/entities/_wrapped.py +0 -0
- /cognite/neat/{rules → _rules}/models/information/__init__.py +0 -0
- /cognite/neat/{store → _store}/__init__.py +0 -0
- /cognite/neat/{utils → _utils}/__init__.py +0 -0
- /cognite/neat/{utils → _utils}/cdf/__init__.py +0 -0
- /cognite/neat/{utils → _utils}/cdf/data_classes.py +0 -0
- /cognite/neat/{utils → _utils}/cdf/loaders/__init__.py +0 -0
- /cognite/neat/{utils → _utils}/cdf/loaders/_base.py +0 -0
- /cognite/neat/{utils → _utils}/collection_.py +0 -0
- /cognite/neat/{utils → _utils}/rdf_.py +0 -0
- /cognite/neat/{utils → _utils}/spreadsheet.py +0 -0
- /cognite/neat/{utils → _utils}/text.py +0 -0
- /cognite/neat/{utils → _utils}/time_.py +0 -0
- /cognite/neat/{utils → _utils}/xml_.py +0 -0
- /cognite/neat/{workflows → _workflows}/examples/Export_Semantic_Data_Model/workflow.yaml +0 -0
- /cognite/neat/{workflows → _workflows}/examples/Import_DMS/workflow.yaml +0 -0
- /cognite/neat/{workflows → _workflows}/examples/Validate_Rules/workflow.yaml +0 -0
- /cognite/neat/{workflows → _workflows}/examples/Validate_Solution_Model/workflow.yaml +0 -0
- /cognite/neat/{workflows → _workflows}/model.py +0 -0
- /cognite/neat/{workflows → _workflows}/steps/__init__.py +0 -0
- /cognite/neat/{workflows → _workflows}/steps/lib/__init__.py +0 -0
- /cognite/neat/{workflows → _workflows}/steps/lib/current/__init__.py +0 -0
- /cognite/neat/{workflows → _workflows}/steps/lib/io/__init__.py +0 -0
- /cognite/neat/{workflows → _workflows}/utils.py +0 -0
- {cognite_neat-0.92.3.dist-info → cognite_neat-0.94.0.dist-info}/LICENSE +0 -0
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
from collections.abc import Iterable
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
from typing import cast
|
|
4
|
+
|
|
5
|
+
from rdflib import URIRef
|
|
6
|
+
from rdflib.util import guess_format
|
|
7
|
+
|
|
8
|
+
from cognite.neat._constants import DEFAULT_BASE_URI
|
|
9
|
+
from cognite.neat._graph._shared import rdflib_to_mime_types
|
|
10
|
+
from cognite.neat._graph.extractors._base import BaseExtractor
|
|
11
|
+
from cognite.neat._graph.models import Triple
|
|
12
|
+
from cognite.neat._issues._base import IssueList
|
|
13
|
+
from cognite.neat._issues.errors import FileNotFoundNeatError, FileTypeUnexpectedError
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class RdfFileExtractor(BaseExtractor):
|
|
17
|
+
"""Extract data from RDF files into Neat.
|
|
18
|
+
|
|
19
|
+
Args:
|
|
20
|
+
filepath (Path): The path to the RDF file.
|
|
21
|
+
mime_type (MIMETypes, optional): The MIME type of the RDF file. Defaults to "application/rdf+xml".
|
|
22
|
+
base_uri (URIRef, optional): The base URI to use. Defaults to None.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
def __init__(
|
|
26
|
+
self,
|
|
27
|
+
filepath: Path,
|
|
28
|
+
base_uri: URIRef = DEFAULT_BASE_URI,
|
|
29
|
+
issue_list: IssueList | None = None,
|
|
30
|
+
):
|
|
31
|
+
self.issue_list = issue_list or IssueList(title=f"{filepath.name}")
|
|
32
|
+
|
|
33
|
+
self.filepath = filepath
|
|
34
|
+
self.mime_type = rdflib_to_mime_types(cast(str, guess_format(str(self.filepath))))
|
|
35
|
+
self.base_uri = base_uri
|
|
36
|
+
|
|
37
|
+
if not self.filepath.exists():
|
|
38
|
+
self.issue_list.append(FileNotFoundNeatError(self.filepath))
|
|
39
|
+
|
|
40
|
+
if not self.mime_type:
|
|
41
|
+
self.issue_list.append(
|
|
42
|
+
FileTypeUnexpectedError(
|
|
43
|
+
self.filepath,
|
|
44
|
+
frozenset([".rdf", ".ttl", ".nt", ".n3", ".owl", ".nq", ".trig"]),
|
|
45
|
+
)
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
def extract(self) -> Iterable[Triple]:
|
|
49
|
+
raise NotImplementedError()
|
|
@@ -6,11 +6,11 @@ from typing import ClassVar, Generic, TypeVar
|
|
|
6
6
|
from cognite.client import CogniteClient
|
|
7
7
|
from cognite.client.data_classes.capabilities import Capability
|
|
8
8
|
|
|
9
|
-
from cognite.neat.
|
|
10
|
-
from cognite.neat.
|
|
11
|
-
from cognite.neat.
|
|
12
|
-
from cognite.neat.
|
|
13
|
-
from cognite.neat.
|
|
9
|
+
from cognite.neat._issues import IssueList, NeatIssue, NeatIssueList
|
|
10
|
+
from cognite.neat._issues.errors import AuthorizationError
|
|
11
|
+
from cognite.neat._store import NeatGraphStore
|
|
12
|
+
from cognite.neat._utils.auxiliary import class_html_doc
|
|
13
|
+
from cognite.neat._utils.upload import UploadResult, UploadResultList
|
|
14
14
|
|
|
15
15
|
T_Output = TypeVar("T_Output")
|
|
16
16
|
|
|
@@ -17,16 +17,17 @@ from cognite.client.data_classes.capabilities import (
|
|
|
17
17
|
)
|
|
18
18
|
from cognite.client.exceptions import CogniteAPIError, CogniteDuplicatedError
|
|
19
19
|
|
|
20
|
-
from cognite.neat.
|
|
21
|
-
from cognite.neat.
|
|
22
|
-
from cognite.neat.
|
|
23
|
-
from cognite.neat.
|
|
24
|
-
from cognite.neat.
|
|
25
|
-
from cognite.neat.
|
|
26
|
-
from cognite.neat.
|
|
27
|
-
from cognite.neat.
|
|
28
|
-
from cognite.neat.
|
|
29
|
-
from cognite.neat.
|
|
20
|
+
from cognite.neat._graph._tracking.base import Tracker
|
|
21
|
+
from cognite.neat._graph._tracking.log import LogTracker
|
|
22
|
+
from cognite.neat._issues import IssueList, NeatError, NeatIssue, NeatIssueList
|
|
23
|
+
from cognite.neat._issues.errors import ResourceCreationError, ResourceNotFoundError
|
|
24
|
+
from cognite.neat._rules._constants import EntityTypes
|
|
25
|
+
from cognite.neat._rules.analysis._asset import AssetAnalysis
|
|
26
|
+
from cognite.neat._rules.models import AssetRules
|
|
27
|
+
from cognite.neat._rules.models.entities import ClassEntity
|
|
28
|
+
from cognite.neat._store import NeatGraphStore
|
|
29
|
+
from cognite.neat._utils.auxiliary import create_sha256_hash
|
|
30
|
+
from cognite.neat._utils.upload import UploadResult
|
|
30
31
|
|
|
31
32
|
from ._base import _END_OF_CLASS, CDFLoader
|
|
32
33
|
|
|
@@ -15,21 +15,21 @@ from cognite.client.exceptions import CogniteAPIError
|
|
|
15
15
|
from pydantic import BaseModel, ValidationInfo, create_model, field_validator
|
|
16
16
|
from rdflib import RDF
|
|
17
17
|
|
|
18
|
-
from cognite.neat.
|
|
19
|
-
from cognite.neat.
|
|
20
|
-
from cognite.neat.
|
|
21
|
-
from cognite.neat.
|
|
18
|
+
from cognite.neat._graph._tracking import LogTracker, Tracker
|
|
19
|
+
from cognite.neat._graph.models import InstanceType
|
|
20
|
+
from cognite.neat._issues import IssueList, NeatIssue, NeatIssueList
|
|
21
|
+
from cognite.neat._issues.errors import (
|
|
22
22
|
ResourceConvertionError,
|
|
23
23
|
ResourceCreationError,
|
|
24
24
|
ResourceDuplicatedError,
|
|
25
25
|
ResourceRetrievalError,
|
|
26
26
|
)
|
|
27
|
-
from cognite.neat.
|
|
28
|
-
from cognite.neat.
|
|
29
|
-
from cognite.neat.
|
|
30
|
-
from cognite.neat.
|
|
31
|
-
from cognite.neat.
|
|
32
|
-
from cognite.neat.
|
|
27
|
+
from cognite.neat._issues.warnings import PropertyTypeNotSupportedWarning
|
|
28
|
+
from cognite.neat._rules.models import DMSRules
|
|
29
|
+
from cognite.neat._rules.models.data_types import _DATA_TYPE_BY_DMS_TYPE, Json
|
|
30
|
+
from cognite.neat._store import NeatGraphStore
|
|
31
|
+
from cognite.neat._utils.auxiliary import create_sha256_hash
|
|
32
|
+
from cognite.neat._utils.upload import UploadResult
|
|
33
33
|
|
|
34
34
|
from ._base import CDFLoader
|
|
35
35
|
|
|
@@ -1,15 +1,18 @@
|
|
|
1
1
|
import warnings
|
|
2
2
|
from collections import defaultdict
|
|
3
|
+
from collections.abc import Iterable
|
|
3
4
|
from typing import Literal, cast, overload
|
|
4
5
|
|
|
5
|
-
from rdflib import RDF, Graph, URIRef
|
|
6
|
+
from rdflib import RDF, Graph, Namespace, URIRef
|
|
6
7
|
from rdflib import Literal as RdfLiteral
|
|
7
8
|
from rdflib.query import ResultRow
|
|
8
9
|
|
|
9
|
-
from cognite.neat.
|
|
10
|
-
from cognite.neat.
|
|
11
|
-
from cognite.neat.
|
|
12
|
-
from cognite.neat.
|
|
10
|
+
from cognite.neat._constants import UNKNOWN_TYPE
|
|
11
|
+
from cognite.neat._graph.models import InstanceType
|
|
12
|
+
from cognite.neat._rules._constants import EntityTypes
|
|
13
|
+
from cognite.neat._rules.models.entities import ClassEntity
|
|
14
|
+
from cognite.neat._rules.models.information import InformationRules
|
|
15
|
+
from cognite.neat._utils.rdf_ import remove_namespace_from_uri
|
|
13
16
|
|
|
14
17
|
from ._construct import build_construct_query
|
|
15
18
|
|
|
@@ -71,30 +74,69 @@ class Queries:
|
|
|
71
74
|
# Select queries gives an iterable of result rows
|
|
72
75
|
return cast(list[ResultRow], list(self.graph.query(query)))
|
|
73
76
|
|
|
74
|
-
def triples_of_type_instances(self, rdf_type: str) -> list[tuple[str, str, str]]:
|
|
77
|
+
def triples_of_type_instances(self, rdf_type: str | URIRef) -> list[tuple[str, str, str]]:
|
|
75
78
|
"""Get all triples of a given type.
|
|
76
79
|
|
|
77
80
|
This method assumes the graph has been transformed into the default namespace.
|
|
78
81
|
"""
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
f"WHERE {{ ?instance a <{self.rules.metadata.namespace[rdf_type]}> . ?instance ?prop ?value . }} "
|
|
84
|
-
"order by ?instance"
|
|
85
|
-
)
|
|
86
|
-
|
|
87
|
-
result = self.graph.query(query)
|
|
88
|
-
|
|
89
|
-
# We cannot include the RDF.type in case there is a neat:type property
|
|
90
|
-
return [remove_namespace_from_uri(*triple) for triple in result if triple[1] != RDF.type] # type: ignore[misc, index]
|
|
82
|
+
if isinstance(rdf_type, URIRef):
|
|
83
|
+
rdf_uri = rdf_type
|
|
84
|
+
elif isinstance(rdf_type, str) and self.rules:
|
|
85
|
+
rdf_uri = self.rules.metadata.namespace[rdf_type]
|
|
91
86
|
else:
|
|
92
87
|
warnings.warn(
|
|
93
|
-
"
|
|
88
|
+
"Unknown namespace. Please either provide a URIRef or set the rules of the store.",
|
|
94
89
|
stacklevel=2,
|
|
95
90
|
)
|
|
96
91
|
return []
|
|
97
92
|
|
|
93
|
+
query = (
|
|
94
|
+
"SELECT ?instance ?prop ?value "
|
|
95
|
+
f"WHERE {{ ?instance a <{rdf_uri}> . ?instance ?prop ?value . }} "
|
|
96
|
+
"order by ?instance"
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
result = self.graph.query(query)
|
|
100
|
+
|
|
101
|
+
# We cannot include the RDF.type in case there is a neat:type property
|
|
102
|
+
return [remove_namespace_from_uri(list(triple)) for triple in result if triple[1] != RDF.type] # type: ignore[misc, index, arg-type]
|
|
103
|
+
|
|
104
|
+
def types_with_property(self, property_uri: URIRef) -> list[URIRef]:
|
|
105
|
+
"""Check if a property exists in the graph store
|
|
106
|
+
|
|
107
|
+
Args:
|
|
108
|
+
property_uri: Property URI to check
|
|
109
|
+
|
|
110
|
+
Returns:
|
|
111
|
+
True if property exists, False otherwise
|
|
112
|
+
"""
|
|
113
|
+
query = f"SELECT DISTINCT ?t WHERE {{ ?s <{property_uri}> ?o ; a ?t}} Limit 1"
|
|
114
|
+
return cast(list[URIRef], [t[0] for t in self.graph.query(query)]) # type: ignore[index]
|
|
115
|
+
|
|
116
|
+
def has_namespace(self, namespace: Namespace) -> bool:
|
|
117
|
+
"""Check if a namespace exists in the graph store
|
|
118
|
+
|
|
119
|
+
Args:
|
|
120
|
+
namespace: Namespace to check
|
|
121
|
+
|
|
122
|
+
Returns:
|
|
123
|
+
True if namespace exists, False otherwise
|
|
124
|
+
"""
|
|
125
|
+
query = f"ASK WHERE {{ ?s ?p ?o . FILTER(STRSTARTS(STR(?p), STR(<{namespace}>))) }}"
|
|
126
|
+
return bool(self.graph.query(query))
|
|
127
|
+
|
|
128
|
+
def has_type(self, type_: URIRef) -> bool:
|
|
129
|
+
"""Check if a type exists in the graph store
|
|
130
|
+
|
|
131
|
+
Args:
|
|
132
|
+
type_: Type to check
|
|
133
|
+
|
|
134
|
+
Returns:
|
|
135
|
+
True if type exists, False otherwise
|
|
136
|
+
"""
|
|
137
|
+
query = f"ASK WHERE {{ ?s a <{type_}> }}"
|
|
138
|
+
return bool(self.graph.query(query))
|
|
139
|
+
|
|
98
140
|
def describe(
|
|
99
141
|
self,
|
|
100
142
|
instance_id: URIRef,
|
|
@@ -249,3 +291,33 @@ class Queries:
|
|
|
249
291
|
if remove_namespace:
|
|
250
292
|
return [remove_namespace_from_uri(res[0]) for res in result]
|
|
251
293
|
return result
|
|
294
|
+
|
|
295
|
+
def multi_value_type_property(
|
|
296
|
+
self,
|
|
297
|
+
) -> Iterable[tuple[URIRef, URIRef, list[URIRef]]]:
|
|
298
|
+
query = """SELECT ?sourceType ?property
|
|
299
|
+
(GROUP_CONCAT(DISTINCT STR(?valueType); SEPARATOR=",") AS ?valueTypes)
|
|
300
|
+
|
|
301
|
+
WHERE {{
|
|
302
|
+
?s ?property ?o .
|
|
303
|
+
?s a ?sourceType .
|
|
304
|
+
OPTIONAL {{ ?o a ?type }}
|
|
305
|
+
|
|
306
|
+
# Key part to determine value type: either object, data or unknown
|
|
307
|
+
BIND( IF(isLiteral(?o),DATATYPE(?o),
|
|
308
|
+
IF(BOUND(?type), ?type,
|
|
309
|
+
<{unknownType}>)) AS ?valueType)
|
|
310
|
+
}}
|
|
311
|
+
|
|
312
|
+
GROUP BY ?sourceType ?property
|
|
313
|
+
HAVING (COUNT(DISTINCT ?valueType) > 1)"""
|
|
314
|
+
|
|
315
|
+
for (
|
|
316
|
+
source_type,
|
|
317
|
+
property_,
|
|
318
|
+
value_types,
|
|
319
|
+
) in cast(
|
|
320
|
+
ResultRow,
|
|
321
|
+
self.graph.query(query.format(unknownType=str(UNKNOWN_TYPE))),
|
|
322
|
+
):
|
|
323
|
+
yield cast(URIRef, source_type), cast(URIRef, property_), [URIRef(uri) for uri in value_types.split(",")]
|
|
@@ -3,17 +3,17 @@ from typing import cast
|
|
|
3
3
|
|
|
4
4
|
from rdflib import Graph, URIRef
|
|
5
5
|
|
|
6
|
-
from cognite.neat.
|
|
7
|
-
from cognite.neat.
|
|
6
|
+
from cognite.neat._rules.analysis import InformationAnalysis
|
|
7
|
+
from cognite.neat._rules.models._rdfpath import (
|
|
8
8
|
Hop,
|
|
9
9
|
RDFPath,
|
|
10
10
|
SelfReferenceProperty,
|
|
11
11
|
SingleProperty,
|
|
12
12
|
Traversal,
|
|
13
13
|
)
|
|
14
|
-
from cognite.neat.
|
|
15
|
-
from cognite.neat.
|
|
16
|
-
from cognite.neat.
|
|
14
|
+
from cognite.neat._rules.models.entities import ClassEntity
|
|
15
|
+
from cognite.neat._rules.models.information import InformationProperty, InformationRules
|
|
16
|
+
from cognite.neat._utils.collection_ import most_occurring_element
|
|
17
17
|
|
|
18
18
|
from ._shared import Triple, hop2property_path
|
|
19
19
|
|
|
@@ -6,12 +6,12 @@ from pydantic import BaseModel, ConfigDict, Field
|
|
|
6
6
|
from rdflib import Graph, Literal, Namespace
|
|
7
7
|
from rdflib.term import URIRef
|
|
8
8
|
|
|
9
|
-
from cognite.neat.
|
|
10
|
-
from cognite.neat.
|
|
9
|
+
from cognite.neat._constants import get_default_prefixes
|
|
10
|
+
from cognite.neat._rules.models._rdfpath import (
|
|
11
11
|
Hop,
|
|
12
12
|
Step,
|
|
13
13
|
)
|
|
14
|
-
from cognite.neat.
|
|
14
|
+
from cognite.neat._utils.rdf_ import remove_namespace_from_uri, uri_to_short_form
|
|
15
15
|
|
|
16
16
|
if sys.version_info >= (3, 11):
|
|
17
17
|
from typing import Self
|
|
@@ -5,8 +5,10 @@ from ._classic_cdf import (
|
|
|
5
5
|
AssetRelationshipConnector,
|
|
6
6
|
AssetSequenceConnector,
|
|
7
7
|
AssetTimeSeriesConnector,
|
|
8
|
+
RelationshipToSchemaTransformer,
|
|
8
9
|
)
|
|
9
10
|
from ._rdfpath import AddSelfReferenceProperty
|
|
11
|
+
from ._value_type import SplitMultiValueProperty
|
|
10
12
|
|
|
11
13
|
__all__ = [
|
|
12
14
|
"AddAssetDepth",
|
|
@@ -16,6 +18,8 @@ __all__ = [
|
|
|
16
18
|
"AssetEventConnector",
|
|
17
19
|
"AssetRelationshipConnector",
|
|
18
20
|
"AddSelfReferenceProperty",
|
|
21
|
+
"SplitMultiValueProperty",
|
|
22
|
+
"RelationshipToSchemaTransformer",
|
|
19
23
|
]
|
|
20
24
|
|
|
21
25
|
Transformers = (
|
|
@@ -26,4 +30,6 @@ Transformers = (
|
|
|
26
30
|
| AssetEventConnector
|
|
27
31
|
| AssetRelationshipConnector
|
|
28
32
|
| AddSelfReferenceProperty
|
|
33
|
+
| SplitMultiValueProperty
|
|
34
|
+
| RelationshipToSchemaTransformer
|
|
29
35
|
)
|
|
@@ -1,9 +1,13 @@
|
|
|
1
|
+
import warnings
|
|
1
2
|
from typing import cast
|
|
2
3
|
|
|
3
|
-
from rdflib import RDF, Graph, Literal, URIRef
|
|
4
|
+
from rdflib import RDF, Graph, Literal, Namespace, URIRef
|
|
5
|
+
from rdflib.query import ResultRow
|
|
4
6
|
|
|
5
|
-
from cognite.neat.
|
|
6
|
-
from cognite.neat.
|
|
7
|
+
from cognite.neat._constants import CLASSIC_CDF_NAMESPACE, DEFAULT_NAMESPACE
|
|
8
|
+
from cognite.neat._graph import extractors
|
|
9
|
+
from cognite.neat._issues.warnings import ResourceNotFoundWarning
|
|
10
|
+
from cognite.neat._utils.rdf_ import remove_namespace_from_uri
|
|
7
11
|
|
|
8
12
|
from ._base import BaseTransformer
|
|
9
13
|
|
|
@@ -313,3 +317,131 @@ class AssetRelationshipConnector(BaseTransformer):
|
|
|
313
317
|
# remove properties that are not needed, specifically the external ids
|
|
314
318
|
graph.remove((relationship_id, self.relationship_source_xid_prop, None))
|
|
315
319
|
graph.remove((relationship_id, self.relationship_target_xid_prop, None))
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
class RelationshipToSchemaTransformer(BaseTransformer):
|
|
323
|
+
"""Replaces relationships with a schema.
|
|
324
|
+
|
|
325
|
+
This transformer analyzes the relationships in the graph and modifies them to be part of the schema
|
|
326
|
+
for Assets, Events, Files, Sequences, and TimeSeries. Relationships without any properties
|
|
327
|
+
are replaced by a simple relationship between the source and target nodes. Relationships with
|
|
328
|
+
properties are replaced by a schema that contains the properties as attributes.
|
|
329
|
+
|
|
330
|
+
Args:
|
|
331
|
+
limit: The minimum number of relationships that need to be present for it
|
|
332
|
+
to be converted into a schema. Default is 1.
|
|
333
|
+
|
|
334
|
+
"""
|
|
335
|
+
|
|
336
|
+
def __init__(self, limit: int = 1, namespace: Namespace = CLASSIC_CDF_NAMESPACE) -> None:
|
|
337
|
+
self._limit = limit
|
|
338
|
+
self._namespace = namespace
|
|
339
|
+
|
|
340
|
+
_NOT_PROPERTIES: frozenset[str] = frozenset(
|
|
341
|
+
{"source_external_id", "target_external_id", "external_id", "source_type", "target_type"}
|
|
342
|
+
)
|
|
343
|
+
_RELATIONSHIP_NODE_TYPES: tuple[str, ...] = tuple(["Asset", "Event", "File", "Sequence", "TimeSeries"])
|
|
344
|
+
description = "Replaces relationships with a schema"
|
|
345
|
+
_use_only_once: bool = True
|
|
346
|
+
_need_changes = frozenset({str(extractors.RelationshipsExtractor.__name__)})
|
|
347
|
+
|
|
348
|
+
_count_by_source_target = """PREFIX classic: <{namespace}>
|
|
349
|
+
|
|
350
|
+
SELECT (COUNT(?instance) AS ?instanceCount)
|
|
351
|
+
WHERE {{
|
|
352
|
+
?instance a classic:Relationship .
|
|
353
|
+
?instance classic:source_type classic:{source_type} .
|
|
354
|
+
?instance classic:target_type classic:{target_type} .
|
|
355
|
+
}}"""
|
|
356
|
+
|
|
357
|
+
_instances = """PREFIX classic: <{namespace}>
|
|
358
|
+
|
|
359
|
+
SELECT ?instance
|
|
360
|
+
WHERE {{
|
|
361
|
+
?instance a classic:Relationship .
|
|
362
|
+
?instance classic:source_type classic:{source_type} .
|
|
363
|
+
?instance classic:target_type classic:{target_type} .
|
|
364
|
+
}}"""
|
|
365
|
+
_lookup_entity_query = """PREFIX classic: <{namespace}>
|
|
366
|
+
|
|
367
|
+
SELECT ?entity
|
|
368
|
+
WHERE {{
|
|
369
|
+
?entity a classic:{entity_type} .
|
|
370
|
+
?entity classic:external_id "{external_id}" .
|
|
371
|
+
}}"""
|
|
372
|
+
|
|
373
|
+
def transform(self, graph: Graph) -> None:
|
|
374
|
+
for source_type in self._RELATIONSHIP_NODE_TYPES:
|
|
375
|
+
for target_type in self._RELATIONSHIP_NODE_TYPES:
|
|
376
|
+
query = self._count_by_source_target.format(
|
|
377
|
+
namespace=self._namespace, source_type=source_type, target_type=target_type
|
|
378
|
+
)
|
|
379
|
+
for instance_count in graph.query(query):
|
|
380
|
+
if int(instance_count[0]) < self._limit: # type: ignore[index, arg-type]
|
|
381
|
+
continue
|
|
382
|
+
query = self._instances.format(
|
|
383
|
+
namespace=self._namespace, source_type=source_type, target_type=target_type
|
|
384
|
+
)
|
|
385
|
+
for result in graph.query(query):
|
|
386
|
+
instance_id = cast(URIRef, result[0]) # type: ignore[index, misc]
|
|
387
|
+
self._convert_relationship_to_schema(graph, instance_id, source_type, target_type)
|
|
388
|
+
|
|
389
|
+
def _convert_relationship_to_schema(
|
|
390
|
+
self, graph: Graph, instance_id: URIRef, source_type: str, target_type: str
|
|
391
|
+
) -> None:
|
|
392
|
+
result = cast(list[ResultRow], list(graph.query(f"DESCRIBE <{instance_id}>")))
|
|
393
|
+
object_by_predicates = cast(
|
|
394
|
+
dict[str, URIRef | Literal], {remove_namespace_from_uri(row[1]): row[2] for row in result}
|
|
395
|
+
)
|
|
396
|
+
source_external_id = cast(URIRef, object_by_predicates["source_external_id"])
|
|
397
|
+
target_source_id = cast(URIRef, object_by_predicates["target_external_id"])
|
|
398
|
+
try:
|
|
399
|
+
source_id = self._lookup_entity(graph, source_type, source_external_id)
|
|
400
|
+
except ValueError:
|
|
401
|
+
warnings.warn(ResourceNotFoundWarning(source_external_id, "class", str(instance_id), "class"), stacklevel=2)
|
|
402
|
+
return None
|
|
403
|
+
try:
|
|
404
|
+
target_id = self._lookup_entity(graph, target_type, target_source_id)
|
|
405
|
+
except ValueError:
|
|
406
|
+
warnings.warn(ResourceNotFoundWarning(target_source_id, "class", str(instance_id), "class"), stacklevel=2)
|
|
407
|
+
return None
|
|
408
|
+
external_id = str(object_by_predicates["external_id"])
|
|
409
|
+
# If there is properties on the relationship, we create a new intermediate node
|
|
410
|
+
self._create_node(graph, object_by_predicates, external_id, source_id, target_id, self._predicate(target_type))
|
|
411
|
+
|
|
412
|
+
for triple in result:
|
|
413
|
+
graph.remove(triple) # type: ignore[arg-type]
|
|
414
|
+
|
|
415
|
+
def _lookup_entity(self, graph: Graph, entity_type: str, external_id: str) -> URIRef:
|
|
416
|
+
query = self._lookup_entity_query.format(
|
|
417
|
+
namespace=self._namespace, entity_type=entity_type, external_id=external_id
|
|
418
|
+
)
|
|
419
|
+
result = list(graph.query(query))
|
|
420
|
+
if len(result) == 1:
|
|
421
|
+
return cast(URIRef, result[0][0]) # type: ignore[index]
|
|
422
|
+
raise ValueError(f"Could not find entity with external_id {external_id} and type {entity_type}")
|
|
423
|
+
|
|
424
|
+
def _create_node(
|
|
425
|
+
self,
|
|
426
|
+
graph: Graph,
|
|
427
|
+
objects_by_predicates: dict[str, URIRef | Literal],
|
|
428
|
+
external_id: str,
|
|
429
|
+
source_id: URIRef,
|
|
430
|
+
target_id: URIRef,
|
|
431
|
+
predicate: URIRef,
|
|
432
|
+
) -> None:
|
|
433
|
+
"""Creates a new intermediate node for the relationship with properties."""
|
|
434
|
+
# Create new node
|
|
435
|
+
instance_id = self._namespace[external_id]
|
|
436
|
+
graph.add((instance_id, RDF.type, self._namespace["Edge"]))
|
|
437
|
+
for prop_name, object_ in objects_by_predicates.items():
|
|
438
|
+
if prop_name in self._NOT_PROPERTIES:
|
|
439
|
+
continue
|
|
440
|
+
graph.add((instance_id, self._namespace[prop_name], object_))
|
|
441
|
+
|
|
442
|
+
# Connect the new node to the source and target nodes
|
|
443
|
+
graph.add((source_id, predicate, instance_id))
|
|
444
|
+
graph.add((instance_id, self._namespace["end_node"], target_id))
|
|
445
|
+
|
|
446
|
+
def _predicate(self, target_type: str) -> URIRef:
|
|
447
|
+
return self._namespace[f"relationship{target_type.capitalize()}"]
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
from rdflib import Namespace
|
|
2
|
+
|
|
3
|
+
from cognite.neat._graph.extractors import IODDExtractor
|
|
4
|
+
|
|
5
|
+
from ._prune_graph import PruneDanglingNodes, TwoHopFlattener
|
|
6
|
+
|
|
7
|
+
IODD = Namespace("http://www.io-link.com/IODD/2010/10/")
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class IODDTwoHopFlattener(TwoHopFlattener):
|
|
11
|
+
_need_changes = frozenset(
|
|
12
|
+
{
|
|
13
|
+
str(IODDExtractor.__name__),
|
|
14
|
+
}
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
def __init__(self):
|
|
18
|
+
super().__init__(destination_node_type=IODD.TextObject, property_predicate=IODD.value, property_name="value")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class IODDPruneDanglingNodes(PruneDanglingNodes):
|
|
22
|
+
_need_changes = frozenset({str(IODDExtractor.__name__), str(IODDTwoHopFlattener.__name__)})
|
|
23
|
+
|
|
24
|
+
def __init__(self):
|
|
25
|
+
super().__init__(node_prune_types=[IODD.TextObject])
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
from rdflib import Graph, Namespace, URIRef
|
|
2
|
+
from rdflib.query import ResultRow
|
|
3
|
+
from rdflib.term import Identifier
|
|
4
|
+
|
|
5
|
+
from ._base import BaseTransformer
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
# TODO: Handle the cse when value is None, which will not make the TextObject resolve
|
|
9
|
+
class TwoHopFlattener(BaseTransformer):
|
|
10
|
+
"""
|
|
11
|
+
Transformer that will flatten the distance between a source node, an intermediate connecting node, and a
|
|
12
|
+
target property that is connected to the intermediate node.
|
|
13
|
+
The transformation result is that the target property is attached directly to the source node, instead of having
|
|
14
|
+
to go via the intermediate node.
|
|
15
|
+
The user can also provide a flag to decide if the intermediate node should be removed from the graph or not
|
|
16
|
+
after connecting the target property to the source node.
|
|
17
|
+
|
|
18
|
+
Ex. TwoHopFlattener:
|
|
19
|
+
|
|
20
|
+
Graph before flattening (with deletion of intermediate node):
|
|
21
|
+
node(A, rdf:type(Pump)) -(predicate("vendor"))>
|
|
22
|
+
node(B, rdf:type(TextObject)) -(predicate("value"))> Literal("CompanyX")
|
|
23
|
+
|
|
24
|
+
Graph after flattening nodes with destination_node_type = rdf:type(TextObject), property_predicate = :value,
|
|
25
|
+
and property_name = "value":
|
|
26
|
+
|
|
27
|
+
node(A, rdf:type(Pump)) -(predicate("vendor"))> Literal("CompanyX")
|
|
28
|
+
|
|
29
|
+
Args:
|
|
30
|
+
destination_node_type: RDF.type of edge Node
|
|
31
|
+
property_predicate: Predicate to use when resolving the value from the edge node
|
|
32
|
+
property_name: name of the property that the intermediate node is pointing to
|
|
33
|
+
delete_connecting_node: bool if the intermediate Node and Edge between source Node
|
|
34
|
+
and target property should be deleted. Defaults to True.
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
description: str = "Prunes the graph of specified node types that do not have connections to other nodes."
|
|
38
|
+
_query_template: str = """SELECT ?sourceNode ?property ?destinationNode ?value WHERE {{
|
|
39
|
+
?sourceNode ?property ?destinationNode .
|
|
40
|
+
?destinationNode a <{destination_node_type}> .
|
|
41
|
+
?destinationNode <{property_predicate}> ?{property_name} . }}"""
|
|
42
|
+
|
|
43
|
+
def __init__(
|
|
44
|
+
self,
|
|
45
|
+
destination_node_type: URIRef,
|
|
46
|
+
property_predicate: Namespace,
|
|
47
|
+
property_name: str,
|
|
48
|
+
delete_connecting_node: bool = True,
|
|
49
|
+
):
|
|
50
|
+
self.destination_node_type = destination_node_type
|
|
51
|
+
self.property_predicate = property_predicate
|
|
52
|
+
self.property_name = property_name
|
|
53
|
+
self.delete_connecting_node = delete_connecting_node
|
|
54
|
+
|
|
55
|
+
def transform(self, graph: Graph) -> None:
|
|
56
|
+
nodes_to_delete: list[Identifier] = []
|
|
57
|
+
|
|
58
|
+
graph_traversals = list(
|
|
59
|
+
graph.query(
|
|
60
|
+
self._query_template.format(
|
|
61
|
+
destination_node_type=self.destination_node_type,
|
|
62
|
+
property_predicate=self.property_predicate,
|
|
63
|
+
property_name=self.property_name,
|
|
64
|
+
)
|
|
65
|
+
)
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
for path in graph_traversals:
|
|
69
|
+
if isinstance(path, ResultRow):
|
|
70
|
+
source_node, predicate, destination_node, property_value = path.asdict().values()
|
|
71
|
+
|
|
72
|
+
# Create new connection from source node to value
|
|
73
|
+
graph.add((source_node, predicate, property_value))
|
|
74
|
+
nodes_to_delete.append(destination_node)
|
|
75
|
+
|
|
76
|
+
if self.delete_connecting_node:
|
|
77
|
+
for node in nodes_to_delete:
|
|
78
|
+
# Remove edge triples to node
|
|
79
|
+
graph.remove((None, None, node))
|
|
80
|
+
# Remove node triple
|
|
81
|
+
graph.remove((node, None, None))
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
class PruneDanglingNodes(BaseTransformer):
|
|
85
|
+
"""
|
|
86
|
+
Knowledge graph pruner and resolver. Will remove rdf triples from graph that does not have connections
|
|
87
|
+
to other nodes, and traverse graph for specified types to resolve the value in the final node and link it to
|
|
88
|
+
the source node.
|
|
89
|
+
|
|
90
|
+
Ex. PruneDanglingNodes
|
|
91
|
+
|
|
92
|
+
Graph before pruning:
|
|
93
|
+
node(A, rdf:type(Pump)) -> node(B, rdf:type(Disc)),
|
|
94
|
+
node(C, rdf:type(Disc))
|
|
95
|
+
|
|
96
|
+
Graph after pruning of nodes rdf:type(Disc):
|
|
97
|
+
|
|
98
|
+
node(A, rd:type(Pump)) -> node(B, rdf:type(Disc))
|
|
99
|
+
|
|
100
|
+
Args:
|
|
101
|
+
node_prune_types: list of RDF types to prune from the Graph if they are stand-alone Nodes
|
|
102
|
+
"""
|
|
103
|
+
|
|
104
|
+
description: str = "Prunes the graph of specified rdf types that do not have connections to other nodes."
|
|
105
|
+
_query_template = """
|
|
106
|
+
SELECT ?subject
|
|
107
|
+
WHERE {{
|
|
108
|
+
?subject a <{rdf_type}> .
|
|
109
|
+
FILTER NOT EXISTS {{ ?s ?p ?subject }}
|
|
110
|
+
}}
|
|
111
|
+
"""
|
|
112
|
+
|
|
113
|
+
def __init__(
|
|
114
|
+
self,
|
|
115
|
+
node_prune_types: list[URIRef],
|
|
116
|
+
):
|
|
117
|
+
self.node_prune_types = node_prune_types
|
|
118
|
+
|
|
119
|
+
def transform(self, graph: Graph) -> None:
|
|
120
|
+
for object_type in self.node_prune_types:
|
|
121
|
+
nodes_without_neighbours = list(graph.query(self._query_template.format(rdf_type=object_type)))
|
|
122
|
+
|
|
123
|
+
for node in nodes_without_neighbours:
|
|
124
|
+
# Remove node and its property triples in the graph
|
|
125
|
+
if isinstance(node, ResultRow):
|
|
126
|
+
graph.remove(triple=(node["subject"], None, None))
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
from rdflib import Graph
|
|
2
2
|
|
|
3
|
-
from cognite.neat.
|
|
4
|
-
from cognite.neat.
|
|
5
|
-
from cognite.neat.
|
|
3
|
+
from cognite.neat._rules.analysis import InformationAnalysis
|
|
4
|
+
from cognite.neat._rules.models._rdfpath import RDFPath, SingleProperty
|
|
5
|
+
from cognite.neat._rules.models.information import InformationRules
|
|
6
6
|
|
|
7
7
|
from ._base import BaseTransformer
|
|
8
8
|
|