cognite-neat 0.106.0__tar.gz → 0.107.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/PKG-INFO +2 -2
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_graph/extractors/__init__.py +5 -1
- cognite_neat-0.107.0/cognite/neat/_graph/extractors/_base.py +58 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_graph/extractors/_classic_cdf/_base.py +16 -3
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_graph/extractors/_classic_cdf/_classic.py +74 -7
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_graph/extractors/_classic_cdf/_relationships.py +2 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_graph/extractors/_classic_cdf/_sequences.py +8 -1
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_graph/extractors/_dms.py +48 -14
- cognite_neat-0.107.0/cognite/neat/_graph/extractors/_dms_graph.py +149 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_graph/extractors/_rdf_file.py +32 -5
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_graph/loaders/_rdf2dms.py +112 -18
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_graph/queries/_construct.py +1 -1
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_graph/transformers/__init__.py +5 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_graph/transformers/_base.py +9 -1
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_graph/transformers/_classic_cdf.py +90 -3
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_graph/transformers/_rdfpath.py +3 -3
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_graph/transformers/_value_type.py +54 -44
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/analysis/_base.py +1 -1
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/analysis/_information.py +14 -13
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/catalog/__init__.py +1 -0
- cognite_neat-0.107.0/cognite/neat/_rules/catalog/classic_model.xlsx +0 -0
- cognite_neat-0.107.0/cognite/neat/_rules/catalog/info-rules-imf.xlsx +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/importers/_dms2rules.py +7 -5
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/importers/_rdf/_inference2rules.py +1 -1
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/models/_base_rules.py +0 -12
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/models/_types.py +5 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/models/dms/_rules.py +50 -2
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/models/information/_rules.py +48 -5
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/models/information/_rules_input.py +1 -1
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/models/mapping/_classic2core.py +4 -5
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/transformers/__init__.py +4 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/transformers/_converters.py +209 -62
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_session/_base.py +2 -6
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_session/_mapping.py +17 -6
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_session/_prepare.py +0 -47
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_session/_read.py +63 -5
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_session/_state.py +7 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_session/_to.py +40 -2
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_session/exceptions.py +7 -3
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_store/_graph_store.py +52 -11
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_store/_rules_store.py +22 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_utils/auth.py +2 -0
- cognite_neat-0.107.0/cognite/neat/_version.py +2 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/pyproject.toml +1 -1
- cognite_neat-0.106.0/cognite/neat/_graph/extractors/_base.py +0 -26
- cognite_neat-0.106.0/cognite/neat/_rules/catalog/info-rules-imf.xlsx +0 -0
- cognite_neat-0.106.0/cognite/neat/_version.py +0 -2
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/LICENSE +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/README.md +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/__init__.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_client/__init__.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_client/_api/__init__.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_client/_api/data_modeling_loaders.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_client/_api/schema.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_client/_api_client.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_client/data_classes/__init__.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_client/data_classes/data_modeling.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_client/data_classes/neat_sequence.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_client/data_classes/schema.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_client/testing.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_config.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_constants.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_graph/__init__.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_graph/_shared.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_graph/_tracking/__init__.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_graph/_tracking/base.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_graph/_tracking/log.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_graph/examples/Knowledge-Graph-Nordic44-dirty.xml +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_graph/examples/Knowledge-Graph-Nordic44.xml +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_graph/examples/__init__.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_graph/examples/skos-capturing-sheet-wind-topics.xlsx +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_graph/extractors/_classic_cdf/__init__.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_graph/extractors/_classic_cdf/_assets.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_graph/extractors/_classic_cdf/_data_sets.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_graph/extractors/_classic_cdf/_events.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_graph/extractors/_classic_cdf/_files.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_graph/extractors/_classic_cdf/_labels.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_graph/extractors/_classic_cdf/_timeseries.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_graph/extractors/_dexpi.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_graph/extractors/_iodd.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_graph/extractors/_mock_graph_generator.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_graph/loaders/__init__.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_graph/loaders/_base.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_graph/queries/__init__.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_graph/queries/_base.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_graph/queries/_shared.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_graph/transformers/_iodd.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_graph/transformers/_prune_graph.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_issues/__init__.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_issues/_base.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_issues/errors/__init__.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_issues/errors/_external.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_issues/errors/_general.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_issues/errors/_properties.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_issues/errors/_resources.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_issues/errors/_workflow.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_issues/formatters.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_issues/warnings/__init__.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_issues/warnings/_external.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_issues/warnings/_general.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_issues/warnings/_models.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_issues/warnings/_properties.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_issues/warnings/_resources.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_issues/warnings/user_modeling.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/__init__.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/_constants.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/_shared.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/analysis/__init__.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/analysis/_dms.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/catalog/hello_world_pump.xlsx +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/exporters/__init__.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/exporters/_base.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/exporters/_rules2dms.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/exporters/_rules2excel.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/exporters/_rules2instance_template.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/exporters/_rules2ontology.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/exporters/_rules2yaml.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/exporters/_validation.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/importers/__init__.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/importers/_base.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/importers/_dtdl2rules/__init__.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/importers/_dtdl2rules/_unit_lookup.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/importers/_dtdl2rules/dtdl_converter.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/importers/_dtdl2rules/dtdl_importer.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/importers/_dtdl2rules/spec.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/importers/_rdf/__init__.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/importers/_rdf/_base.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/importers/_rdf/_imf2rules.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/importers/_rdf/_owl2rules.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/importers/_rdf/_shared.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/importers/_spreadsheet2rules.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/importers/_yaml2rules.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/models/__init__.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/models/_base_input.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/models/_rdfpath.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/models/data_types.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/models/dms/__init__.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/models/dms/_exporter.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/models/dms/_rules_input.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/models/dms/_validation.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/models/entities/__init__.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/models/entities/_constants.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/models/entities/_loaders.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/models/entities/_multi_value.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/models/entities/_single_value.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/models/entities/_types.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/models/entities/_wrapped.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/models/information/__init__.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/models/information/_validation.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/models/mapping/__init__.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/models/mapping/_classic2core.yaml +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/transformers/_base.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/transformers/_mapping.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_rules/transformers/_verification.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_session/__init__.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_session/_collector.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_session/_drop.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_session/_inspect.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_session/_set.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_session/_show.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_session/_wizard.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_session/engine/__init__.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_session/engine/_import.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_session/engine/_interface.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_session/engine/_load.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_shared.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_store/__init__.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_store/_provenance.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_store/exceptions.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_utils/__init__.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_utils/auxiliary.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_utils/collection_.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_utils/graph_transformations_report.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_utils/io_.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_utils/rdf_.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_utils/reader/__init__.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_utils/reader/_base.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_utils/spreadsheet.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_utils/text.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_utils/time_.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_utils/upload.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_utils/xml_.py +0 -0
- {cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/py.typed +0 -0
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: cognite-neat
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.107.0
|
|
4
4
|
Summary: Knowledge graph transformation
|
|
5
|
-
Home-page: https://cognite-neat.readthedocs-hosted.com/
|
|
6
5
|
License: Apache-2.0
|
|
7
6
|
Author: Nikola Vasiljevic
|
|
8
7
|
Author-email: nikola.vasiljevic@cognite.com
|
|
@@ -54,6 +53,7 @@ Requires-Dist: tomli (>=2.0.1,<3.0.0) ; python_version < "3.11"
|
|
|
54
53
|
Requires-Dist: typing_extensions (>=4.8,<5.0) ; python_version < "3.11"
|
|
55
54
|
Requires-Dist: urllib3 (>=2,<3)
|
|
56
55
|
Project-URL: Documentation, https://cognite-neat.readthedocs-hosted.com/
|
|
56
|
+
Project-URL: Homepage, https://cognite-neat.readthedocs-hosted.com/
|
|
57
57
|
Project-URL: Repository, https://github.com/cognitedata/neat
|
|
58
58
|
Description-Content-Type: text/markdown
|
|
59
59
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
from cognite.neat._session.engine._interface import Extractor as EngineExtractor
|
|
2
2
|
|
|
3
|
-
from ._base import BaseExtractor
|
|
3
|
+
from ._base import BaseExtractor, KnowledgeGraphExtractor
|
|
4
4
|
from ._classic_cdf._assets import AssetsExtractor
|
|
5
5
|
from ._classic_cdf._classic import ClassicGraphExtractor
|
|
6
6
|
from ._classic_cdf._data_sets import DataSetExtractor
|
|
@@ -12,6 +12,7 @@ from ._classic_cdf._sequences import SequencesExtractor
|
|
|
12
12
|
from ._classic_cdf._timeseries import TimeSeriesExtractor
|
|
13
13
|
from ._dexpi import DexpiExtractor
|
|
14
14
|
from ._dms import DMSExtractor
|
|
15
|
+
from ._dms_graph import DMSGraphExtractor
|
|
15
16
|
from ._iodd import IODDExtractor
|
|
16
17
|
from ._mock_graph_generator import MockGraphGenerator
|
|
17
18
|
from ._rdf_file import RdfFileExtractor
|
|
@@ -21,11 +22,13 @@ __all__ = [
|
|
|
21
22
|
"BaseExtractor",
|
|
22
23
|
"ClassicGraphExtractor",
|
|
23
24
|
"DMSExtractor",
|
|
25
|
+
"DMSGraphExtractor",
|
|
24
26
|
"DataSetExtractor",
|
|
25
27
|
"DexpiExtractor",
|
|
26
28
|
"EventsExtractor",
|
|
27
29
|
"FilesExtractor",
|
|
28
30
|
"IODDExtractor",
|
|
31
|
+
"KnowledgeGraphExtractor",
|
|
29
32
|
"LabelsExtractor",
|
|
30
33
|
"MockGraphGenerator",
|
|
31
34
|
"RdfFileExtractor",
|
|
@@ -51,6 +54,7 @@ TripleExtractors = (
|
|
|
51
54
|
| ClassicGraphExtractor
|
|
52
55
|
| DataSetExtractor
|
|
53
56
|
| EngineExtractor
|
|
57
|
+
| DMSGraphExtractor
|
|
54
58
|
)
|
|
55
59
|
|
|
56
60
|
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
from abc import abstractmethod
|
|
2
|
+
from collections.abc import Iterable
|
|
3
|
+
from typing import TYPE_CHECKING
|
|
4
|
+
|
|
5
|
+
from rdflib import URIRef
|
|
6
|
+
|
|
7
|
+
from cognite.neat._constants import DEFAULT_NAMESPACE
|
|
8
|
+
from cognite.neat._rules.models import InformationRules
|
|
9
|
+
from cognite.neat._shared import Triple
|
|
10
|
+
from cognite.neat._utils.auxiliary import class_html_doc
|
|
11
|
+
|
|
12
|
+
if TYPE_CHECKING:
|
|
13
|
+
from cognite.neat._store._provenance import Agent as ProvenanceAgent
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class BaseExtractor:
|
|
17
|
+
"""This is the base class for all extractors. It defines the interface that
|
|
18
|
+
extractors must implement.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
def _get_activity_names(self) -> list[str]:
|
|
22
|
+
"""Returns the name of the activities that the extractor performs,
|
|
23
|
+
i.e., the actions that it performs when you call extract().."""
|
|
24
|
+
# This method can be overridden by subclasses that runs multiple extractors
|
|
25
|
+
# for example the ClassicGraphExtractor
|
|
26
|
+
return [type(self).__name__]
|
|
27
|
+
|
|
28
|
+
@abstractmethod
|
|
29
|
+
def extract(self) -> Iterable[Triple]:
|
|
30
|
+
raise NotImplementedError()
|
|
31
|
+
|
|
32
|
+
@classmethod
|
|
33
|
+
def _repr_html_(cls) -> str:
|
|
34
|
+
return class_html_doc(cls)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class KnowledgeGraphExtractor(BaseExtractor):
|
|
38
|
+
"""A knowledge graph extractor extracts triples with a schema"""
|
|
39
|
+
|
|
40
|
+
@abstractmethod
|
|
41
|
+
def get_information_rules(self) -> InformationRules:
|
|
42
|
+
"""Returns the information rules that the extractor uses."""
|
|
43
|
+
raise NotImplementedError()
|
|
44
|
+
|
|
45
|
+
@property
|
|
46
|
+
def description(self) -> str:
|
|
47
|
+
return self.__doc__.strip().split("\n")[0] if self.__doc__ else "Missing"
|
|
48
|
+
|
|
49
|
+
@property
|
|
50
|
+
def source_uri(self) -> URIRef:
|
|
51
|
+
raise NotImplementedError
|
|
52
|
+
|
|
53
|
+
@property
|
|
54
|
+
def agent(self) -> "ProvenanceAgent":
|
|
55
|
+
"""Provenance agent for the importer."""
|
|
56
|
+
from cognite.neat._store._provenance import Agent as ProvenanceAgent
|
|
57
|
+
|
|
58
|
+
return ProvenanceAgent(id_=DEFAULT_NAMESPACE[f"agent/{type(self).__name__}"])
|
{cognite_neat-0.106.0 → cognite_neat-0.107.0}/cognite/neat/_graph/extractors/_classic_cdf/_base.py
RENAMED
|
@@ -89,6 +89,7 @@ class ClassicCDFBaseExtractor(BaseExtractor, ABC, Generic[T_CogniteResource]):
|
|
|
89
89
|
skip_metadata_values: Set[str] | None = DEFAULT_SKIP_METADATA_VALUES,
|
|
90
90
|
camel_case: bool = True,
|
|
91
91
|
as_write: bool = False,
|
|
92
|
+
prefix: str | None = None,
|
|
92
93
|
):
|
|
93
94
|
self.namespace = namespace or DEFAULT_NAMESPACE
|
|
94
95
|
self.items = items
|
|
@@ -99,6 +100,7 @@ class ClassicCDFBaseExtractor(BaseExtractor, ABC, Generic[T_CogniteResource]):
|
|
|
99
100
|
self.skip_metadata_values = skip_metadata_values
|
|
100
101
|
self.camel_case = camel_case
|
|
101
102
|
self.as_write = as_write
|
|
103
|
+
self.prefix = prefix
|
|
102
104
|
|
|
103
105
|
def extract(self) -> Iterable[Triple]:
|
|
104
106
|
"""Extracts an asset with the given asset_id."""
|
|
@@ -174,6 +176,8 @@ class ClassicCDFBaseExtractor(BaseExtractor, ABC, Generic[T_CogniteResource]):
|
|
|
174
176
|
type_ = self._default_rdf_type
|
|
175
177
|
if self.to_type:
|
|
176
178
|
type_ = self.to_type(item) or type_
|
|
179
|
+
if self.prefix:
|
|
180
|
+
type_ = f"{self.prefix}{type_}"
|
|
177
181
|
return self._SPACE_PATTERN.sub("_", type_)
|
|
178
182
|
|
|
179
183
|
def _as_object(self, raw: Any, key: str) -> Literal | URIRef:
|
|
@@ -218,9 +222,12 @@ class ClassicCDFBaseExtractor(BaseExtractor, ABC, Generic[T_CogniteResource]):
|
|
|
218
222
|
skip_metadata_values: Set[str] | None = DEFAULT_SKIP_METADATA_VALUES,
|
|
219
223
|
camel_case: bool = True,
|
|
220
224
|
as_write: bool = False,
|
|
225
|
+
prefix: str | None = None,
|
|
221
226
|
):
|
|
222
227
|
total, items = cls._handle_no_access(lambda: cls._from_dataset(client, data_set_external_id))
|
|
223
|
-
return cls(
|
|
228
|
+
return cls(
|
|
229
|
+
items, namespace, to_type, total, limit, unpack_metadata, skip_metadata_values, camel_case, as_write, prefix
|
|
230
|
+
)
|
|
224
231
|
|
|
225
232
|
@classmethod
|
|
226
233
|
@abstractmethod
|
|
@@ -241,9 +248,12 @@ class ClassicCDFBaseExtractor(BaseExtractor, ABC, Generic[T_CogniteResource]):
|
|
|
241
248
|
skip_metadata_values: Set[str] | None = DEFAULT_SKIP_METADATA_VALUES,
|
|
242
249
|
camel_case: bool = True,
|
|
243
250
|
as_write: bool = False,
|
|
251
|
+
prefix: str | None = None,
|
|
244
252
|
):
|
|
245
253
|
total, items = cls._handle_no_access(lambda: cls._from_hierarchy(client, root_asset_external_id))
|
|
246
|
-
return cls(
|
|
254
|
+
return cls(
|
|
255
|
+
items, namespace, to_type, total, limit, unpack_metadata, skip_metadata_values, camel_case, as_write, prefix
|
|
256
|
+
)
|
|
247
257
|
|
|
248
258
|
@classmethod
|
|
249
259
|
@abstractmethod
|
|
@@ -263,9 +273,12 @@ class ClassicCDFBaseExtractor(BaseExtractor, ABC, Generic[T_CogniteResource]):
|
|
|
263
273
|
skip_metadata_values: Set[str] | None = DEFAULT_SKIP_METADATA_VALUES,
|
|
264
274
|
camel_case: bool = True,
|
|
265
275
|
as_write: bool = False,
|
|
276
|
+
prefix: str | None = None,
|
|
266
277
|
):
|
|
267
278
|
total, items = cls._from_file(file_path)
|
|
268
|
-
return cls(
|
|
279
|
+
return cls(
|
|
280
|
+
items, namespace, to_type, total, limit, unpack_metadata, skip_metadata_values, camel_case, as_write, prefix
|
|
281
|
+
)
|
|
269
282
|
|
|
270
283
|
@classmethod
|
|
271
284
|
@abstractmethod
|
|
@@ -1,18 +1,25 @@
|
|
|
1
1
|
import warnings
|
|
2
2
|
from collections import defaultdict
|
|
3
3
|
from collections.abc import Iterable, Sequence
|
|
4
|
-
from typing import ClassVar, NamedTuple
|
|
4
|
+
from typing import ClassVar, NamedTuple, cast
|
|
5
5
|
|
|
6
6
|
from cognite.client import CogniteClient
|
|
7
7
|
from cognite.client.exceptions import CogniteAPIError
|
|
8
|
-
from rdflib import Namespace
|
|
8
|
+
from rdflib import Namespace, URIRef
|
|
9
9
|
|
|
10
|
-
from cognite.neat._constants import CLASSIC_CDF_NAMESPACE
|
|
11
|
-
from cognite.neat._graph.extractors._base import
|
|
10
|
+
from cognite.neat._constants import CLASSIC_CDF_NAMESPACE, DEFAULT_NAMESPACE, get_default_prefixes_and_namespaces
|
|
11
|
+
from cognite.neat._graph.extractors._base import KnowledgeGraphExtractor
|
|
12
|
+
from cognite.neat._issues.errors import NeatValueError
|
|
12
13
|
from cognite.neat._issues.warnings import CDFAuthWarning
|
|
14
|
+
from cognite.neat._rules._shared import ReadRules
|
|
15
|
+
from cognite.neat._rules.catalog import classic_model
|
|
16
|
+
from cognite.neat._rules.models import InformationInputRules, InformationRules
|
|
17
|
+
from cognite.neat._rules.models._rdfpath import Entity as RDFPathEntity
|
|
18
|
+
from cognite.neat._rules.models._rdfpath import RDFPath, SingleProperty
|
|
13
19
|
from cognite.neat._shared import Triple
|
|
14
20
|
from cognite.neat._utils.collection_ import chunker, iterate_progress_bar
|
|
15
21
|
from cognite.neat._utils.rdf_ import remove_namespace_from_uri
|
|
22
|
+
from cognite.neat._utils.text import to_snake
|
|
16
23
|
|
|
17
24
|
from ._assets import AssetsExtractor
|
|
18
25
|
from ._base import ClassicCDFBaseExtractor, InstanceIdPrefix
|
|
@@ -37,7 +44,7 @@ class _ClassicCoreType(NamedTuple):
|
|
|
37
44
|
api_name: str
|
|
38
45
|
|
|
39
46
|
|
|
40
|
-
class ClassicGraphExtractor(
|
|
47
|
+
class ClassicGraphExtractor(KnowledgeGraphExtractor):
|
|
41
48
|
"""This extractor extracts all classic CDF Resources.
|
|
42
49
|
|
|
43
50
|
The Classic Graph consists of the following core resource type.
|
|
@@ -93,6 +100,7 @@ class ClassicGraphExtractor(BaseExtractor):
|
|
|
93
100
|
root_asset_external_id: str | None = None,
|
|
94
101
|
namespace: Namespace | None = None,
|
|
95
102
|
limit_per_type: int | None = None,
|
|
103
|
+
prefix: str | None = None,
|
|
96
104
|
):
|
|
97
105
|
self._client = client
|
|
98
106
|
if sum([bool(data_set_external_id), bool(root_asset_external_id)]) != 1:
|
|
@@ -101,8 +109,14 @@ class ClassicGraphExtractor(BaseExtractor):
|
|
|
101
109
|
self._data_set_external_id = data_set_external_id
|
|
102
110
|
self._namespace = namespace or CLASSIC_CDF_NAMESPACE
|
|
103
111
|
self._extractor_args = dict(
|
|
104
|
-
namespace=self._namespace,
|
|
112
|
+
namespace=self._namespace,
|
|
113
|
+
unpack_metadata=False,
|
|
114
|
+
as_write=True,
|
|
115
|
+
camel_case=True,
|
|
116
|
+
limit=limit_per_type,
|
|
117
|
+
prefix=prefix,
|
|
105
118
|
)
|
|
119
|
+
self._prefix = prefix
|
|
106
120
|
self._limit_per_type = limit_per_type
|
|
107
121
|
|
|
108
122
|
self._source_external_ids_by_type: dict[InstanceIdPrefix, set[str]] = defaultdict(set)
|
|
@@ -144,6 +158,59 @@ class ClassicGraphExtractor(BaseExtractor):
|
|
|
144
158
|
else:
|
|
145
159
|
self._extracted_data_sets = True
|
|
146
160
|
|
|
161
|
+
def get_information_rules(self) -> InformationRules:
|
|
162
|
+
# To avoid circular imports
|
|
163
|
+
from cognite.neat._rules.importers import ExcelImporter
|
|
164
|
+
|
|
165
|
+
unverified = cast(ReadRules[InformationInputRules], ExcelImporter(classic_model).to_rules())
|
|
166
|
+
if unverified.rules is None:
|
|
167
|
+
raise NeatValueError(f"Could not read the classic model rules from {classic_model}.")
|
|
168
|
+
|
|
169
|
+
verified = unverified.rules.as_verified_rules()
|
|
170
|
+
prefixes = get_default_prefixes_and_namespaces()
|
|
171
|
+
instance_prefix: str | None = next((k for k, v in prefixes.items() if v == self._namespace), None)
|
|
172
|
+
if instance_prefix is None:
|
|
173
|
+
# We need to add a new prefix
|
|
174
|
+
instance_prefix = f"prefix_{len(prefixes) + 1}"
|
|
175
|
+
prefixes[instance_prefix] = self._namespace
|
|
176
|
+
verified.prefixes = prefixes
|
|
177
|
+
|
|
178
|
+
is_snake_case = self._extractor_args["camel_case"] is False
|
|
179
|
+
for prop in verified.properties:
|
|
180
|
+
prop_id = prop.property_
|
|
181
|
+
if is_snake_case:
|
|
182
|
+
prop_id = to_snake(prop_id)
|
|
183
|
+
prop.instance_source = RDFPath(
|
|
184
|
+
traversal=SingleProperty(
|
|
185
|
+
class_=RDFPathEntity(prefix=instance_prefix, suffix=prop.class_.suffix),
|
|
186
|
+
property=RDFPathEntity(prefix=instance_prefix, suffix=prop_id),
|
|
187
|
+
)
|
|
188
|
+
)
|
|
189
|
+
return verified
|
|
190
|
+
|
|
191
|
+
@property
|
|
192
|
+
def description(self) -> str:
|
|
193
|
+
if self._data_set_external_id:
|
|
194
|
+
source = f"data set {self._data_set_external_id}."
|
|
195
|
+
elif self._root_asset_external_id:
|
|
196
|
+
source = f"root asset {self._root_asset_external_id}."
|
|
197
|
+
else:
|
|
198
|
+
source = "unknown source."
|
|
199
|
+
return f"Extracting clasic CDF Graph (Assets, TimeSeries, Sequences, Events, Files) from {source}."
|
|
200
|
+
|
|
201
|
+
@property
|
|
202
|
+
def source_uri(self) -> URIRef:
|
|
203
|
+
if self._data_set_external_id:
|
|
204
|
+
resource = "dataset"
|
|
205
|
+
external_id = self._data_set_external_id
|
|
206
|
+
elif self._root_asset_external_id:
|
|
207
|
+
resource = "asset"
|
|
208
|
+
external_id = self._root_asset_external_id
|
|
209
|
+
else:
|
|
210
|
+
resource = "unknown"
|
|
211
|
+
external_id = "unknown"
|
|
212
|
+
return DEFAULT_NAMESPACE[f"{self._client.config.project}/{resource}/{external_id}"]
|
|
213
|
+
|
|
147
214
|
def _extract_core_start_nodes(self):
|
|
148
215
|
for core_node in self._classic_node_types:
|
|
149
216
|
if self._data_set_external_id:
|
|
@@ -217,7 +284,7 @@ class ClassicGraphExtractor(BaseExtractor):
|
|
|
217
284
|
self._source_external_ids_by_type[resource_type].add(remove_namespace_from_uri(triple[2]))
|
|
218
285
|
elif triple[1] == self._namespace.labels:
|
|
219
286
|
self._labels.add(remove_namespace_from_uri(triple[2]).removeprefix(InstanceIdPrefix.label))
|
|
220
|
-
elif triple[1] == self._namespace.
|
|
287
|
+
elif triple[1] == self._namespace.dataSetId:
|
|
221
288
|
self._data_set_ids.add(
|
|
222
289
|
int(remove_namespace_from_uri(triple[2]).removeprefix(InstanceIdPrefix.data_set))
|
|
223
290
|
)
|
|
@@ -28,6 +28,7 @@ class RelationshipsExtractor(ClassicCDFBaseExtractor[Relationship]):
|
|
|
28
28
|
skip_metadata_values: Set[str] | None = DEFAULT_SKIP_METADATA_VALUES,
|
|
29
29
|
camel_case: bool = True,
|
|
30
30
|
as_write: bool = False,
|
|
31
|
+
prefix: str | None = None,
|
|
31
32
|
):
|
|
32
33
|
super().__init__(
|
|
33
34
|
items,
|
|
@@ -39,6 +40,7 @@ class RelationshipsExtractor(ClassicCDFBaseExtractor[Relationship]):
|
|
|
39
40
|
skip_metadata_values=skip_metadata_values,
|
|
40
41
|
camel_case=camel_case,
|
|
41
42
|
as_write=as_write,
|
|
43
|
+
prefix=prefix,
|
|
42
44
|
)
|
|
43
45
|
# This is used by the ClassicExtractor to log the target nodes, such
|
|
44
46
|
# that it can extract them.
|
|
@@ -52,10 +52,11 @@ class SequencesExtractor(ClassicCDFBaseExtractor[NeatSequence]):
|
|
|
52
52
|
skip_metadata_values: Set[str] | None = DEFAULT_SKIP_METADATA_VALUES,
|
|
53
53
|
camel_case: bool = True,
|
|
54
54
|
as_write: bool = False,
|
|
55
|
+
prefix: str | None = None,
|
|
55
56
|
unpack_columns: bool = False,
|
|
56
57
|
):
|
|
57
58
|
super().__init__(
|
|
58
|
-
items, namespace, to_type, total, limit, unpack_metadata, skip_metadata_values, camel_case, as_write
|
|
59
|
+
items, namespace, to_type, total, limit, unpack_metadata, skip_metadata_values, camel_case, as_write, prefix
|
|
59
60
|
)
|
|
60
61
|
self.unpack_columns = unpack_columns
|
|
61
62
|
|
|
@@ -71,6 +72,7 @@ class SequencesExtractor(ClassicCDFBaseExtractor[NeatSequence]):
|
|
|
71
72
|
skip_metadata_values: Set[str] | None = DEFAULT_SKIP_METADATA_VALUES,
|
|
72
73
|
camel_case: bool = True,
|
|
73
74
|
as_write: bool = False,
|
|
75
|
+
prefix: str | None = None,
|
|
74
76
|
unpack_columns: bool = False,
|
|
75
77
|
):
|
|
76
78
|
total, items = cls._handle_no_access(lambda: cls._from_dataset(client, data_set_external_id))
|
|
@@ -84,6 +86,7 @@ class SequencesExtractor(ClassicCDFBaseExtractor[NeatSequence]):
|
|
|
84
86
|
skip_metadata_values,
|
|
85
87
|
camel_case,
|
|
86
88
|
as_write,
|
|
89
|
+
prefix,
|
|
87
90
|
unpack_columns,
|
|
88
91
|
)
|
|
89
92
|
|
|
@@ -99,6 +102,7 @@ class SequencesExtractor(ClassicCDFBaseExtractor[NeatSequence]):
|
|
|
99
102
|
skip_metadata_values: Set[str] | None = DEFAULT_SKIP_METADATA_VALUES,
|
|
100
103
|
camel_case: bool = True,
|
|
101
104
|
as_write: bool = False,
|
|
105
|
+
prefix: str | None = None,
|
|
102
106
|
unpack_columns: bool = False,
|
|
103
107
|
):
|
|
104
108
|
total, items = cls._handle_no_access(lambda: cls._from_hierarchy(client, root_asset_external_id))
|
|
@@ -112,6 +116,7 @@ class SequencesExtractor(ClassicCDFBaseExtractor[NeatSequence]):
|
|
|
112
116
|
skip_metadata_values,
|
|
113
117
|
camel_case,
|
|
114
118
|
as_write,
|
|
119
|
+
prefix,
|
|
115
120
|
unpack_columns,
|
|
116
121
|
)
|
|
117
122
|
|
|
@@ -126,6 +131,7 @@ class SequencesExtractor(ClassicCDFBaseExtractor[NeatSequence]):
|
|
|
126
131
|
skip_metadata_values: Set[str] | None = DEFAULT_SKIP_METADATA_VALUES,
|
|
127
132
|
camel_case: bool = True,
|
|
128
133
|
as_write: bool = False,
|
|
134
|
+
prefix: str | None = None,
|
|
129
135
|
unpack_columns: bool = False,
|
|
130
136
|
):
|
|
131
137
|
total, items = cls._from_file(file_path)
|
|
@@ -139,6 +145,7 @@ class SequencesExtractor(ClassicCDFBaseExtractor[NeatSequence]):
|
|
|
139
145
|
skip_metadata_values,
|
|
140
146
|
camel_case,
|
|
141
147
|
as_write,
|
|
148
|
+
prefix,
|
|
142
149
|
unpack_columns,
|
|
143
150
|
)
|
|
144
151
|
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import urllib.parse
|
|
1
2
|
from collections.abc import Iterable, Iterator
|
|
2
3
|
from typing import cast
|
|
3
4
|
|
|
@@ -5,6 +6,7 @@ from cognite.client import CogniteClient
|
|
|
5
6
|
from cognite.client import data_modeling as dm
|
|
6
7
|
from cognite.client.data_classes.data_modeling import DataModelIdentifier
|
|
7
8
|
from cognite.client.data_classes.data_modeling.instances import Instance, PropertyValue
|
|
9
|
+
from cognite.client.utils.useful_types import SequenceNotStr
|
|
8
10
|
from rdflib import RDF, XSD, Literal, Namespace, URIRef
|
|
9
11
|
|
|
10
12
|
from cognite.neat._constants import DEFAULT_SPACE_URI
|
|
@@ -38,7 +40,12 @@ class DMSExtractor(BaseExtractor):
|
|
|
38
40
|
|
|
39
41
|
@classmethod
|
|
40
42
|
def from_data_model(
|
|
41
|
-
cls,
|
|
43
|
+
cls,
|
|
44
|
+
client: CogniteClient,
|
|
45
|
+
data_model: DataModelIdentifier,
|
|
46
|
+
limit: int | None = None,
|
|
47
|
+
overwrite_namespace: Namespace | None = None,
|
|
48
|
+
instance_space: str | SequenceNotStr[str] | None = None,
|
|
42
49
|
) -> "DMSExtractor":
|
|
43
50
|
"""Create an extractor from a data model.
|
|
44
51
|
|
|
@@ -46,22 +53,38 @@ class DMSExtractor(BaseExtractor):
|
|
|
46
53
|
client: The Cognite client to use.
|
|
47
54
|
data_model: The data model to extract.
|
|
48
55
|
limit: The maximum number of instances to extract.
|
|
56
|
+
overwrite_namespace: If provided, this will overwrite the space of the extracted items.
|
|
57
|
+
instance_space: The space to extract instances from.
|
|
49
58
|
"""
|
|
50
59
|
retrieved = client.data_modeling.data_models.retrieve(data_model, inline_views=True)
|
|
51
60
|
if not retrieved:
|
|
52
61
|
raise ResourceRetrievalError(dm.DataModelId.load(data_model), "data model", "Data Model is missing in CDF")
|
|
53
|
-
return cls.from_views(client, retrieved.latest_version().views, limit)
|
|
62
|
+
return cls.from_views(client, retrieved.latest_version().views, limit, overwrite_namespace, instance_space)
|
|
54
63
|
|
|
55
64
|
@classmethod
|
|
56
|
-
def from_views(
|
|
65
|
+
def from_views(
|
|
66
|
+
cls,
|
|
67
|
+
client: CogniteClient,
|
|
68
|
+
views: Iterable[dm.View],
|
|
69
|
+
limit: int | None = None,
|
|
70
|
+
overwrite_namespace: Namespace | None = None,
|
|
71
|
+
instance_space: str | SequenceNotStr[str] | None = None,
|
|
72
|
+
) -> "DMSExtractor":
|
|
57
73
|
"""Create an extractor from a set of views.
|
|
58
74
|
|
|
59
75
|
Args:
|
|
60
76
|
client: The Cognite client to use.
|
|
61
77
|
views: The views to extract.
|
|
62
78
|
limit: The maximum number of instances to extract.
|
|
79
|
+
overwrite_namespace: If provided, this will overwrite the space of the extracted items.
|
|
80
|
+
instance_space: The space to extract instances from.
|
|
63
81
|
"""
|
|
64
|
-
return cls(
|
|
82
|
+
return cls(
|
|
83
|
+
_InstanceIterator(client, views, instance_space),
|
|
84
|
+
total=None,
|
|
85
|
+
limit=limit,
|
|
86
|
+
overwrite_namespace=overwrite_namespace,
|
|
87
|
+
)
|
|
65
88
|
|
|
66
89
|
def extract(self) -> Iterable[Triple]:
|
|
67
90
|
for count, item in enumerate(self.items, 1):
|
|
@@ -105,6 +128,10 @@ class DMSExtractor(BaseExtractor):
|
|
|
105
128
|
else:
|
|
106
129
|
raise NotImplementedError(f"Unknown instance type {type(instance)}")
|
|
107
130
|
|
|
131
|
+
if self.overwrite_namespace:
|
|
132
|
+
# If the namespace is overwritten, keep the original space as a property to avoid losing information.
|
|
133
|
+
yield id_, self._get_namespace(instance.space)["space"], Literal(instance.space)
|
|
134
|
+
|
|
108
135
|
for view_id, properties in instance.properties.items():
|
|
109
136
|
namespace = self._get_namespace(view_id.space)
|
|
110
137
|
for key, value in properties.items():
|
|
@@ -124,35 +151,42 @@ class DMSExtractor(BaseExtractor):
|
|
|
124
151
|
yield from self._get_objects(item)
|
|
125
152
|
|
|
126
153
|
def _as_uri_ref(self, instance: Instance | dm.DirectRelationReference) -> URIRef:
|
|
127
|
-
return self._get_namespace(instance.space)[instance.external_id]
|
|
154
|
+
return self._get_namespace(instance.space)[urllib.parse.quote(instance.external_id)]
|
|
128
155
|
|
|
129
156
|
def _get_namespace(self, space: str) -> Namespace:
|
|
130
157
|
if self.overwrite_namespace:
|
|
131
158
|
return self.overwrite_namespace
|
|
132
|
-
return Namespace(DEFAULT_SPACE_URI.format(space=space))
|
|
159
|
+
return Namespace(DEFAULT_SPACE_URI.format(space=urllib.parse.quote(space)))
|
|
133
160
|
|
|
134
161
|
|
|
135
|
-
class _InstanceIterator(
|
|
136
|
-
def __init__(
|
|
162
|
+
class _InstanceIterator(Iterable[Instance]):
|
|
163
|
+
def __init__(
|
|
164
|
+
self, client: CogniteClient, views: Iterable[dm.View], instance_space: str | SequenceNotStr[str] | None = None
|
|
165
|
+
):
|
|
137
166
|
self.client = client
|
|
138
167
|
self.views = views
|
|
168
|
+
self.instance_space = instance_space
|
|
139
169
|
|
|
140
170
|
def __iter__(self) -> Iterator[Instance]:
|
|
141
|
-
return self
|
|
142
|
-
|
|
143
|
-
def __next__(self) -> Instance: # type: ignore[misc]
|
|
144
171
|
for view in self.views:
|
|
172
|
+
view_id = view.as_id()
|
|
145
173
|
# All nodes and edges with properties
|
|
146
|
-
|
|
147
|
-
|
|
174
|
+
if view.used_for in ("node", "all"):
|
|
175
|
+
yield from self.client.data_modeling.instances(
|
|
176
|
+
chunk_size=None, instance_type="node", sources=[view_id], space=self.instance_space
|
|
177
|
+
)
|
|
178
|
+
if view.used_for in ("edge", "all"):
|
|
179
|
+
yield from self.client.data_modeling.instances(
|
|
180
|
+
chunk_size=None, instance_type="edge", sources=[view_id], space=self.instance_space
|
|
181
|
+
)
|
|
148
182
|
|
|
149
183
|
for prop in view.properties.values():
|
|
150
184
|
if isinstance(prop, dm.EdgeConnection):
|
|
151
|
-
# Get all edges with properties
|
|
152
185
|
yield from self.client.data_modeling.instances(
|
|
153
186
|
chunk_size=None,
|
|
154
187
|
instance_type="edge",
|
|
155
188
|
filter=dm.filters.Equals(
|
|
156
189
|
["edge", "type"], {"space": prop.type.space, "externalId": prop.type.external_id}
|
|
157
190
|
),
|
|
191
|
+
space=self.instance_space,
|
|
158
192
|
)
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
from collections.abc import Iterable, Sequence
|
|
2
|
+
|
|
3
|
+
from cognite.client import data_modeling as dm
|
|
4
|
+
from cognite.client.exceptions import CogniteAPIError
|
|
5
|
+
from cognite.client.utils.useful_types import SequenceNotStr
|
|
6
|
+
from rdflib import Namespace, URIRef
|
|
7
|
+
|
|
8
|
+
from cognite.neat._client import NeatClient
|
|
9
|
+
from cognite.neat._constants import DEFAULT_NAMESPACE
|
|
10
|
+
from cognite.neat._issues import IssueList, NeatIssue, catch_warnings
|
|
11
|
+
from cognite.neat._issues.warnings import CDFAuthWarning, ResourceNotFoundWarning, ResourceRetrievalWarning
|
|
12
|
+
from cognite.neat._rules.importers import DMSImporter
|
|
13
|
+
from cognite.neat._rules.models import DMSRules, InformationRules
|
|
14
|
+
from cognite.neat._rules.transformers import DMSToInformation, VerifyDMSRules
|
|
15
|
+
from cognite.neat._shared import Triple
|
|
16
|
+
|
|
17
|
+
from ._base import KnowledgeGraphExtractor
|
|
18
|
+
from ._dms import DMSExtractor
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class DMSGraphExtractor(KnowledgeGraphExtractor):
|
|
22
|
+
def __init__(
|
|
23
|
+
self,
|
|
24
|
+
data_model: dm.DataModel[dm.View],
|
|
25
|
+
client: NeatClient,
|
|
26
|
+
namespace: Namespace = DEFAULT_NAMESPACE,
|
|
27
|
+
issues: Sequence[NeatIssue] | None = None,
|
|
28
|
+
instance_space: str | SequenceNotStr[str] | None = None,
|
|
29
|
+
) -> None:
|
|
30
|
+
self._client = client
|
|
31
|
+
self._data_model = data_model
|
|
32
|
+
self._namespace = namespace or DEFAULT_NAMESPACE
|
|
33
|
+
self._issues = IssueList(issues)
|
|
34
|
+
self._instance_space = instance_space
|
|
35
|
+
|
|
36
|
+
self._views: list[dm.View] | None = None
|
|
37
|
+
self._information_rules: InformationRules | None = None
|
|
38
|
+
self._dms_rules: DMSRules | None = None
|
|
39
|
+
|
|
40
|
+
@classmethod
|
|
41
|
+
def from_data_model_id(
|
|
42
|
+
cls,
|
|
43
|
+
data_model_id: dm.DataModelIdentifier,
|
|
44
|
+
client: NeatClient,
|
|
45
|
+
namespace: Namespace = DEFAULT_NAMESPACE,
|
|
46
|
+
instance_space: str | SequenceNotStr[str] | None = None,
|
|
47
|
+
) -> "DMSGraphExtractor":
|
|
48
|
+
issues: list[NeatIssue] = []
|
|
49
|
+
try:
|
|
50
|
+
data_model = client.data_modeling.data_models.retrieve(data_model_id, inline_views=True)
|
|
51
|
+
except CogniteAPIError as e:
|
|
52
|
+
issues.append(CDFAuthWarning("retrieving data model", str(e)))
|
|
53
|
+
return cls(
|
|
54
|
+
cls._create_empty_model(dm.DataModelId.load(data_model_id)), client, namespace, issues, instance_space
|
|
55
|
+
)
|
|
56
|
+
if not data_model:
|
|
57
|
+
issues.append(ResourceRetrievalWarning(frozenset({data_model_id}), "data model"))
|
|
58
|
+
return cls(
|
|
59
|
+
cls._create_empty_model(dm.DataModelId.load(data_model_id)), client, namespace, issues, instance_space
|
|
60
|
+
)
|
|
61
|
+
return cls(data_model.latest_version(), client, namespace, issues, instance_space)
|
|
62
|
+
|
|
63
|
+
@classmethod
|
|
64
|
+
def _create_empty_model(cls, data_model_id: dm.DataModelId) -> dm.DataModel:
|
|
65
|
+
return dm.DataModel(
|
|
66
|
+
data_model_id.space,
|
|
67
|
+
data_model_id.external_id,
|
|
68
|
+
data_model_id.version or "MISSING",
|
|
69
|
+
is_global=False,
|
|
70
|
+
last_updated_time=0,
|
|
71
|
+
created_time=0,
|
|
72
|
+
description=None,
|
|
73
|
+
name=None,
|
|
74
|
+
views=[],
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
@property
|
|
78
|
+
def _model_views(self) -> list[dm.View]:
|
|
79
|
+
if self._views is None:
|
|
80
|
+
self._views = self._get_views()
|
|
81
|
+
return self._views
|
|
82
|
+
|
|
83
|
+
@property
|
|
84
|
+
def description(self) -> str:
|
|
85
|
+
return "Extracts a data model with nodes and edges."
|
|
86
|
+
|
|
87
|
+
@property
|
|
88
|
+
def source_uri(self) -> URIRef:
|
|
89
|
+
space, external_id, version = self._data_model.as_id().as_tuple()
|
|
90
|
+
return DEFAULT_NAMESPACE[f"{self._client.config.project}/{space}/{external_id}/{version}"]
|
|
91
|
+
|
|
92
|
+
def extract(self) -> Iterable[Triple]:
|
|
93
|
+
"""Extracts the knowledge graph from the data model."""
|
|
94
|
+
views = self._model_views
|
|
95
|
+
yield from DMSExtractor.from_views(
|
|
96
|
+
self._client,
|
|
97
|
+
views,
|
|
98
|
+
overwrite_namespace=self._namespace,
|
|
99
|
+
instance_space=self._instance_space,
|
|
100
|
+
).extract()
|
|
101
|
+
|
|
102
|
+
def _get_views(self) -> list[dm.View]:
|
|
103
|
+
view_by_id: dict[dm.ViewId, dm.View] = {}
|
|
104
|
+
if view_ids := [view_id for view_id in self._data_model.views if isinstance(view_id, dm.ViewId)]:
|
|
105
|
+
try:
|
|
106
|
+
# MyPy does not understand the isinstance check above.
|
|
107
|
+
retrieved = self._client.data_modeling.views.retrieve(ids=view_ids) # type: ignore[arg-type]
|
|
108
|
+
except CogniteAPIError as e:
|
|
109
|
+
self._issues.append(CDFAuthWarning("retrieving views", str(e)))
|
|
110
|
+
else:
|
|
111
|
+
view_by_id.update({view.as_id(): view for view in retrieved})
|
|
112
|
+
views: list[dm.View] = []
|
|
113
|
+
data_model_id = self._data_model.as_id()
|
|
114
|
+
for dm_view in self._data_model.views:
|
|
115
|
+
if isinstance(dm_view, dm.View):
|
|
116
|
+
views.append(dm_view)
|
|
117
|
+
elif isinstance(dm_view, dm.ViewId):
|
|
118
|
+
if view := view_by_id.get(dm_view):
|
|
119
|
+
views.append(view)
|
|
120
|
+
else:
|
|
121
|
+
self._issues.append(ResourceNotFoundWarning(dm_view, "view", data_model_id, "data model"))
|
|
122
|
+
return views
|
|
123
|
+
|
|
124
|
+
def get_information_rules(self) -> InformationRules:
|
|
125
|
+
"""Returns the information rules that the extractor uses."""
|
|
126
|
+
if self._information_rules is None:
|
|
127
|
+
self._information_rules, self._dms_rules = self._create_rules()
|
|
128
|
+
return self._information_rules
|
|
129
|
+
|
|
130
|
+
def get_dms_rules(self) -> DMSRules:
|
|
131
|
+
"""Returns the DMS rules that the extractor uses."""
|
|
132
|
+
if self._dms_rules is None:
|
|
133
|
+
self._information_rules, self._dms_rules = self._create_rules()
|
|
134
|
+
return self._dms_rules
|
|
135
|
+
|
|
136
|
+
def get_issues(self) -> IssueList:
|
|
137
|
+
"""Returns the issues that occurred during the extraction."""
|
|
138
|
+
return self._issues
|
|
139
|
+
|
|
140
|
+
def _create_rules(self) -> tuple[InformationRules, DMSRules]:
|
|
141
|
+
# The DMS and Information rules must be created together to link them property.
|
|
142
|
+
importer = DMSImporter.from_data_model(self._client, self._data_model)
|
|
143
|
+
unverified_dms = importer.to_rules()
|
|
144
|
+
with catch_warnings() as issues:
|
|
145
|
+
# Any errors occur will be raised and caught outside the extractor.
|
|
146
|
+
verified_dms = VerifyDMSRules(client=self._client).transform(unverified_dms)
|
|
147
|
+
information_rules = DMSToInformation(self._namespace).transform(verified_dms)
|
|
148
|
+
self._issues.extend(issues)
|
|
149
|
+
return information_rules, verified_dms
|