cognite-neat 0.123.2__py3-none-any.whl → 0.127.30__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cognite/neat/__init__.py +2 -2
- cognite/neat/_client/__init__.py +4 -0
- cognite/neat/_client/api.py +8 -0
- cognite/neat/_client/client.py +21 -0
- cognite/neat/_client/config.py +40 -0
- cognite/neat/_client/containers_api.py +125 -0
- cognite/neat/_client/data_classes.py +44 -0
- cognite/neat/_client/data_model_api.py +115 -0
- cognite/neat/_client/spaces_api.py +115 -0
- cognite/neat/_client/statistics_api.py +24 -0
- cognite/neat/_client/views_api.py +129 -0
- cognite/neat/_config.py +185 -0
- cognite/neat/_data_model/_analysis.py +196 -0
- cognite/neat/_data_model/_constants.py +67 -0
- cognite/neat/_data_model/_identifiers.py +61 -0
- cognite/neat/_data_model/_shared.py +41 -0
- cognite/neat/_data_model/deployer/_differ.py +140 -0
- cognite/neat/_data_model/deployer/_differ_container.py +360 -0
- cognite/neat/_data_model/deployer/_differ_data_model.py +54 -0
- cognite/neat/_data_model/deployer/_differ_space.py +9 -0
- cognite/neat/_data_model/deployer/_differ_view.py +299 -0
- cognite/neat/_data_model/deployer/data_classes.py +529 -0
- cognite/neat/_data_model/deployer/deployer.py +401 -0
- cognite/neat/_data_model/exporters/__init__.py +15 -0
- cognite/neat/_data_model/exporters/_api_exporter.py +37 -0
- cognite/neat/_data_model/exporters/_base.py +24 -0
- cognite/neat/_data_model/exporters/_table_exporter/exporter.py +128 -0
- cognite/neat/_data_model/exporters/_table_exporter/workbook.py +409 -0
- cognite/neat/_data_model/exporters/_table_exporter/writer.py +421 -0
- cognite/neat/_data_model/importers/__init__.py +5 -0
- cognite/neat/_data_model/importers/_api_importer.py +166 -0
- cognite/neat/_data_model/importers/_base.py +16 -0
- cognite/neat/_data_model/importers/_table_importer/data_classes.py +295 -0
- cognite/neat/_data_model/importers/_table_importer/importer.py +192 -0
- cognite/neat/_data_model/importers/_table_importer/reader.py +1063 -0
- cognite/neat/_data_model/importers/_table_importer/source.py +94 -0
- cognite/neat/_data_model/models/conceptual/_base.py +18 -0
- cognite/neat/_data_model/models/conceptual/_concept.py +67 -0
- cognite/neat/_data_model/models/conceptual/_data_model.py +51 -0
- cognite/neat/_data_model/models/conceptual/_properties.py +104 -0
- cognite/neat/_data_model/models/conceptual/_property.py +105 -0
- cognite/neat/_data_model/models/dms/__init__.py +206 -0
- cognite/neat/_data_model/models/dms/_base.py +31 -0
- cognite/neat/_data_model/models/dms/_constants.py +48 -0
- cognite/neat/_data_model/models/dms/_constraints.py +42 -0
- cognite/neat/_data_model/models/dms/_container.py +159 -0
- cognite/neat/_data_model/models/dms/_data_model.py +95 -0
- cognite/neat/_data_model/models/dms/_data_types.py +195 -0
- cognite/neat/_data_model/models/dms/_http.py +28 -0
- cognite/neat/_data_model/models/dms/_indexes.py +30 -0
- cognite/neat/_data_model/models/dms/_limits.py +96 -0
- cognite/neat/_data_model/models/dms/_references.py +135 -0
- cognite/neat/_data_model/models/dms/_schema.py +18 -0
- cognite/neat/_data_model/models/dms/_space.py +48 -0
- cognite/neat/_data_model/models/dms/_types.py +17 -0
- cognite/neat/_data_model/models/dms/_view_filter.py +282 -0
- cognite/neat/_data_model/models/dms/_view_property.py +235 -0
- cognite/neat/_data_model/models/dms/_views.py +210 -0
- cognite/neat/_data_model/models/entities/__init__.py +50 -0
- cognite/neat/_data_model/models/entities/_base.py +101 -0
- cognite/neat/_data_model/models/entities/_constants.py +22 -0
- cognite/neat/_data_model/models/entities/_data_types.py +144 -0
- cognite/neat/_data_model/models/entities/_identifiers.py +61 -0
- cognite/neat/_data_model/models/entities/_parser.py +226 -0
- cognite/neat/_data_model/validation/dms/__init__.py +75 -0
- cognite/neat/_data_model/validation/dms/_ai_readiness.py +364 -0
- cognite/neat/_data_model/validation/dms/_base.py +307 -0
- cognite/neat/_data_model/validation/dms/_connections.py +638 -0
- cognite/neat/_data_model/validation/dms/_consistency.py +57 -0
- cognite/neat/_data_model/validation/dms/_containers.py +174 -0
- cognite/neat/_data_model/validation/dms/_limits.py +420 -0
- cognite/neat/_data_model/validation/dms/_orchestrator.py +222 -0
- cognite/neat/_data_model/validation/dms/_views.py +103 -0
- cognite/neat/_exceptions.py +56 -0
- cognite/neat/_issues.py +68 -0
- cognite/neat/_session/__init__.py +3 -0
- cognite/neat/_session/_html/_render.py +30 -0
- cognite/neat/_session/_html/static/__init__.py +8 -0
- cognite/neat/_session/_html/static/deployment.css +303 -0
- cognite/neat/_session/_html/static/deployment.js +150 -0
- cognite/neat/_session/_html/static/issues.css +211 -0
- cognite/neat/_session/_html/static/issues.js +168 -0
- cognite/neat/_session/_html/static/shared.css +186 -0
- cognite/neat/_session/_html/templates/__init__.py +4 -0
- cognite/neat/_session/_html/templates/deployment.html +75 -0
- cognite/neat/_session/_html/templates/issues.html +45 -0
- cognite/neat/_session/_issues.py +81 -0
- cognite/neat/_session/_opt.py +35 -0
- cognite/neat/_session/_physical.py +261 -0
- cognite/neat/_session/_result.py +236 -0
- cognite/neat/_session/_session.py +88 -0
- cognite/neat/_session/_usage_analytics/__init__.py +0 -0
- cognite/neat/_session/_usage_analytics/_collector.py +131 -0
- cognite/neat/_session/_usage_analytics/_constants.py +23 -0
- cognite/neat/_session/_usage_analytics/_storage.py +240 -0
- cognite/neat/_session/_wrappers.py +82 -0
- cognite/neat/_state_machine/__init__.py +10 -0
- cognite/neat/_state_machine/_base.py +37 -0
- cognite/neat/_state_machine/_states.py +52 -0
- cognite/neat/_store/__init__.py +3 -0
- cognite/neat/_store/_provenance.py +81 -0
- cognite/neat/_store/_store.py +156 -0
- cognite/neat/_utils/__init__.py +0 -0
- cognite/neat/_utils/_reader.py +194 -0
- cognite/neat/_utils/auxiliary.py +39 -0
- cognite/neat/_utils/collection.py +11 -0
- cognite/neat/_utils/http_client/__init__.py +39 -0
- cognite/neat/_utils/http_client/_client.py +245 -0
- cognite/neat/_utils/http_client/_config.py +19 -0
- cognite/neat/_utils/http_client/_data_classes.py +294 -0
- cognite/neat/_utils/http_client/_tracker.py +31 -0
- cognite/neat/_utils/text.py +71 -0
- cognite/neat/_utils/useful_types.py +37 -0
- cognite/neat/_utils/validation.py +154 -0
- cognite/neat/_version.py +1 -1
- cognite/neat/v0/__init__.py +0 -0
- cognite/neat/v0/core/__init__.py +0 -0
- cognite/neat/v0/core/_client/_api/__init__.py +0 -0
- cognite/neat/{core → v0/core}/_client/_api/data_modeling_loaders.py +86 -7
- cognite/neat/{core → v0/core}/_client/_api/neat_instances.py +5 -5
- cognite/neat/{core → v0/core}/_client/_api/schema.py +5 -5
- cognite/neat/{core → v0/core}/_client/_api/statistics.py +3 -3
- cognite/neat/{core → v0/core}/_client/_api_client.py +1 -1
- cognite/neat/v0/core/_client/data_classes/__init__.py +0 -0
- cognite/neat/{core → v0/core}/_client/data_classes/schema.py +4 -4
- cognite/neat/{core → v0/core}/_client/testing.py +1 -1
- cognite/neat/{core → v0/core}/_constants.py +10 -3
- cognite/neat/v0/core/_data_model/__init__.py +0 -0
- cognite/neat/{core → v0/core}/_data_model/_constants.py +9 -6
- cognite/neat/{core → v0/core}/_data_model/_shared.py +5 -5
- cognite/neat/{core → v0/core}/_data_model/analysis/_base.py +12 -8
- cognite/neat/{core → v0/core}/_data_model/exporters/__init__.py +1 -2
- cognite/neat/{core → v0/core}/_data_model/exporters/_base.py +7 -7
- cognite/neat/{core → v0/core}/_data_model/exporters/_data_model2dms.py +9 -9
- cognite/neat/{core → v0/core}/_data_model/exporters/_data_model2excel.py +13 -13
- cognite/neat/{core → v0/core}/_data_model/exporters/_data_model2instance_template.py +4 -4
- cognite/neat/{core/_data_model/exporters/_data_model2ontology.py → v0/core/_data_model/exporters/_data_model2semantic_model.py} +126 -133
- cognite/neat/{core → v0/core}/_data_model/exporters/_data_model2yaml.py +1 -1
- cognite/neat/{core → v0/core}/_data_model/importers/__init__.py +4 -6
- cognite/neat/{core → v0/core}/_data_model/importers/_base.py +5 -5
- cognite/neat/{core → v0/core}/_data_model/importers/_base_file_reader.py +2 -2
- cognite/neat/{core → v0/core}/_data_model/importers/_dict2data_model.py +6 -6
- cognite/neat/{core → v0/core}/_data_model/importers/_dms2data_model.py +19 -16
- cognite/neat/v0/core/_data_model/importers/_graph2data_model.py +299 -0
- cognite/neat/v0/core/_data_model/importers/_rdf/__init__.py +4 -0
- cognite/neat/{core → v0/core}/_data_model/importers/_rdf/_base.py +13 -13
- cognite/neat/{core → v0/core}/_data_model/importers/_rdf/_inference2rdata_model.py +14 -14
- cognite/neat/v0/core/_data_model/importers/_rdf/_owl2data_model.py +144 -0
- cognite/neat/v0/core/_data_model/importers/_rdf/_shared.py +255 -0
- cognite/neat/{core → v0/core}/_data_model/importers/_spreadsheet2data_model.py +94 -13
- cognite/neat/{core → v0/core}/_data_model/models/__init__.py +3 -3
- cognite/neat/{core → v0/core}/_data_model/models/_base_verified.py +5 -5
- cognite/neat/v0/core/_data_model/models/_import_contexts.py +82 -0
- cognite/neat/{core → v0/core}/_data_model/models/_types.py +5 -5
- cognite/neat/{core → v0/core}/_data_model/models/conceptual/_unverified.py +18 -12
- cognite/neat/v0/core/_data_model/models/conceptual/_validation.py +308 -0
- cognite/neat/{core → v0/core}/_data_model/models/conceptual/_verified.py +13 -11
- cognite/neat/{core → v0/core}/_data_model/models/data_types.py +14 -4
- cognite/neat/{core → v0/core}/_data_model/models/entities/__init__.py +6 -0
- cognite/neat/v0/core/_data_model/models/entities/_loaders.py +155 -0
- cognite/neat/{core → v0/core}/_data_model/models/entities/_multi_value.py +2 -2
- cognite/neat/v0/core/_data_model/models/entities/_restrictions.py +230 -0
- cognite/neat/{core → v0/core}/_data_model/models/entities/_single_value.py +121 -16
- cognite/neat/{core → v0/core}/_data_model/models/entities/_types.py +10 -0
- cognite/neat/{core → v0/core}/_data_model/models/mapping/_classic2core.py +5 -5
- cognite/neat/{core → v0/core}/_data_model/models/physical/__init__.py +1 -1
- cognite/neat/{core → v0/core}/_data_model/models/physical/_exporter.py +28 -21
- cognite/neat/{core → v0/core}/_data_model/models/physical/_unverified.py +141 -38
- cognite/neat/{core → v0/core}/_data_model/models/physical/_validation.py +190 -24
- cognite/neat/{core → v0/core}/_data_model/models/physical/_verified.py +135 -15
- cognite/neat/{core → v0/core}/_data_model/transformers/__init__.py +2 -0
- cognite/neat/{core → v0/core}/_data_model/transformers/_base.py +4 -4
- cognite/neat/{core → v0/core}/_data_model/transformers/_converters.py +39 -32
- cognite/neat/{core → v0/core}/_data_model/transformers/_mapping.py +7 -7
- cognite/neat/v0/core/_data_model/transformers/_union_conceptual.py +208 -0
- cognite/neat/{core → v0/core}/_data_model/transformers/_verification.py +7 -7
- cognite/neat/v0/core/_instances/__init__.py +0 -0
- cognite/neat/{core → v0/core}/_instances/_tracking/base.py +1 -1
- cognite/neat/{core → v0/core}/_instances/_tracking/log.py +1 -1
- cognite/neat/{core → v0/core}/_instances/extractors/__init__.py +1 -1
- cognite/neat/{core → v0/core}/_instances/extractors/_base.py +6 -6
- cognite/neat/v0/core/_instances/extractors/_classic_cdf/__init__.py +0 -0
- cognite/neat/{core → v0/core}/_instances/extractors/_classic_cdf/_base.py +7 -7
- cognite/neat/{core → v0/core}/_instances/extractors/_classic_cdf/_classic.py +12 -12
- cognite/neat/{core → v0/core}/_instances/extractors/_classic_cdf/_relationships.py +3 -3
- cognite/neat/{core → v0/core}/_instances/extractors/_classic_cdf/_sequences.py +2 -2
- cognite/neat/{core → v0/core}/_instances/extractors/_dict.py +6 -3
- cognite/neat/{core → v0/core}/_instances/extractors/_dms.py +6 -6
- cognite/neat/{core → v0/core}/_instances/extractors/_dms_graph.py +11 -11
- cognite/neat/{core → v0/core}/_instances/extractors/_mock_graph_generator.py +10 -10
- cognite/neat/{core → v0/core}/_instances/extractors/_raw.py +3 -3
- cognite/neat/{core → v0/core}/_instances/extractors/_rdf_file.py +7 -7
- cognite/neat/{core → v0/core}/_instances/loaders/_base.py +5 -5
- cognite/neat/{core → v0/core}/_instances/loaders/_rdf2dms.py +17 -17
- cognite/neat/{core → v0/core}/_instances/loaders/_rdf_to_instance_space.py +11 -11
- cognite/neat/{core → v0/core}/_instances/queries/_select.py +29 -3
- cognite/neat/{core → v0/core}/_instances/queries/_update.py +1 -1
- cognite/neat/{core → v0/core}/_instances/transformers/_base.py +4 -4
- cognite/neat/{core → v0/core}/_instances/transformers/_classic_cdf.py +6 -6
- cognite/neat/{core → v0/core}/_instances/transformers/_prune_graph.py +4 -4
- cognite/neat/{core → v0/core}/_instances/transformers/_rdfpath.py +1 -1
- cognite/neat/{core → v0/core}/_instances/transformers/_value_type.py +4 -4
- cognite/neat/{core → v0/core}/_issues/_base.py +11 -6
- cognite/neat/{core → v0/core}/_issues/_contextmanagers.py +8 -6
- cognite/neat/{core → v0/core}/_issues/_factory.py +11 -8
- cognite/neat/{core → v0/core}/_issues/errors/__init__.py +3 -1
- cognite/neat/{core → v0/core}/_issues/errors/_external.py +1 -1
- cognite/neat/{core → v0/core}/_issues/errors/_general.py +1 -1
- cognite/neat/{core → v0/core}/_issues/errors/_properties.py +12 -1
- cognite/neat/{core → v0/core}/_issues/errors/_resources.py +2 -2
- cognite/neat/{core → v0/core}/_issues/errors/_wrapper.py +7 -3
- cognite/neat/{core → v0/core}/_issues/warnings/__init__.py +5 -1
- cognite/neat/{core → v0/core}/_issues/warnings/_external.py +1 -1
- cognite/neat/{core → v0/core}/_issues/warnings/_general.py +1 -1
- cognite/neat/{core → v0/core}/_issues/warnings/_models.py +39 -4
- cognite/neat/{core → v0/core}/_issues/warnings/_properties.py +13 -2
- cognite/neat/{core → v0/core}/_issues/warnings/_resources.py +1 -1
- cognite/neat/{core → v0/core}/_issues/warnings/user_modeling.py +1 -1
- cognite/neat/{core → v0/core}/_store/_data_model.py +13 -12
- cognite/neat/{core → v0/core}/_store/_instance.py +45 -12
- cognite/neat/{core → v0/core}/_store/_provenance.py +3 -3
- cognite/neat/{core → v0/core}/_store/exceptions.py +4 -4
- cognite/neat/v0/core/_utils/__init__.py +0 -0
- cognite/neat/{core → v0/core}/_utils/auth.py +1 -1
- cognite/neat/{core → v0/core}/_utils/auxiliary.py +7 -1
- cognite/neat/{core → v0/core}/_utils/collection_.py +2 -2
- cognite/neat/{core → v0/core}/_utils/graph_transformations_report.py +1 -1
- cognite/neat/{core → v0/core}/_utils/rdf_.py +38 -14
- cognite/neat/{core → v0/core}/_utils/reader/_base.py +1 -1
- cognite/neat/{core → v0/core}/_utils/spreadsheet.py +22 -4
- cognite/neat/v0/core/_utils/tarjan.py +44 -0
- cognite/neat/{core → v0/core}/_utils/text.py +1 -1
- cognite/neat/{core → v0/core}/_utils/upload.py +3 -3
- cognite/neat/v0/plugins/__init__.py +4 -0
- cognite/neat/v0/plugins/_base.py +9 -0
- cognite/neat/v0/plugins/_data_model.py +48 -0
- cognite/neat/{plugins → v0/plugins}/_issues.py +1 -1
- cognite/neat/{plugins → v0/plugins}/_manager.py +7 -16
- cognite/neat/{session → v0/session}/_base.py +13 -10
- cognite/neat/{session → v0/session}/_collector.py +1 -1
- cognite/neat/v0/session/_diff.py +51 -0
- cognite/neat/{session → v0/session}/_drop.py +3 -3
- cognite/neat/{session → v0/session}/_explore.py +2 -2
- cognite/neat/{session → v0/session}/_fix.py +2 -2
- cognite/neat/{session → v0/session}/_inspect.py +3 -3
- cognite/neat/{session → v0/session}/_mapping.py +3 -3
- cognite/neat/{session → v0/session}/_plugin.py +4 -5
- cognite/neat/{session → v0/session}/_prepare.py +8 -8
- cognite/neat/{session → v0/session}/_read.py +33 -43
- cognite/neat/{session → v0/session}/_set.py +8 -8
- cognite/neat/{session → v0/session}/_show.py +5 -5
- cognite/neat/{session → v0/session}/_state.py +22 -8
- cognite/neat/{session → v0/session}/_subset.py +4 -4
- cognite/neat/{session → v0/session}/_template.py +11 -11
- cognite/neat/{session → v0/session}/_to.py +12 -12
- cognite/neat/{session → v0/session}/_wizard.py +1 -1
- cognite/neat/{session → v0/session}/engine/_load.py +1 -1
- cognite/neat/{session → v0/session}/exceptions.py +5 -5
- cognite/neat/v1.py +3 -0
- {cognite_neat-0.123.2.dist-info → cognite_neat-0.127.30.dist-info}/METADATA +9 -8
- cognite_neat-0.127.30.dist-info/RECORD +319 -0
- {cognite_neat-0.123.2.dist-info → cognite_neat-0.127.30.dist-info}/WHEEL +1 -1
- cognite/neat/core/_data_model/importers/_dtdl2data_model/__init__.py +0 -3
- cognite/neat/core/_data_model/importers/_dtdl2data_model/_unit_lookup.py +0 -224
- cognite/neat/core/_data_model/importers/_dtdl2data_model/dtdl_converter.py +0 -320
- cognite/neat/core/_data_model/importers/_dtdl2data_model/dtdl_importer.py +0 -155
- cognite/neat/core/_data_model/importers/_dtdl2data_model/spec.py +0 -363
- cognite/neat/core/_data_model/importers/_rdf/__init__.py +0 -5
- cognite/neat/core/_data_model/importers/_rdf/_imf2data_model.py +0 -98
- cognite/neat/core/_data_model/importers/_rdf/_owl2data_model.py +0 -87
- cognite/neat/core/_data_model/importers/_rdf/_shared.py +0 -168
- cognite/neat/core/_data_model/models/conceptual/_validation.py +0 -294
- cognite/neat/core/_data_model/models/entities/_loaders.py +0 -75
- cognite/neat/plugins/__init__.py +0 -3
- cognite/neat/plugins/data_model/importers/__init__.py +0 -5
- cognite/neat/plugins/data_model/importers/_base.py +0 -28
- cognite_neat-0.123.2.dist-info/RECORD +0 -197
- /cognite/neat/{core → _data_model}/__init__.py +0 -0
- /cognite/neat/{core/_client/_api → _data_model/deployer}/__init__.py +0 -0
- /cognite/neat/{core/_client/data_classes → _data_model/exporters/_table_exporter}/__init__.py +0 -0
- /cognite/neat/{core/_data_model → _data_model/importers/_table_importer}/__init__.py +0 -0
- /cognite/neat/{core/_instances → _data_model/models}/__init__.py +0 -0
- /cognite/neat/{core/_instances/extractors/_classic_cdf → _data_model/models/conceptual}/__init__.py +0 -0
- /cognite/neat/{core/_utils → _data_model/validation}/__init__.py +0 -0
- /cognite/neat/{plugins/data_model → _session/_html}/__init__.py +0 -0
- /cognite/neat/{core → v0/core}/_client/__init__.py +0 -0
- /cognite/neat/{core → v0/core}/_client/data_classes/data_modeling.py +0 -0
- /cognite/neat/{core → v0/core}/_client/data_classes/neat_sequence.py +0 -0
- /cognite/neat/{core → v0/core}/_client/data_classes/statistics.py +0 -0
- /cognite/neat/{core → v0/core}/_config.py +0 -0
- /cognite/neat/{core → v0/core}/_data_model/analysis/__init__.py +0 -0
- /cognite/neat/{core → v0/core}/_data_model/catalog/__init__.py +0 -0
- /cognite/neat/{core → v0/core}/_data_model/catalog/classic_model.xlsx +0 -0
- /cognite/neat/{core → v0/core}/_data_model/catalog/conceptual-imf-data-model.xlsx +0 -0
- /cognite/neat/{core → v0/core}/_data_model/catalog/hello_world_pump.xlsx +0 -0
- /cognite/neat/{core → v0/core}/_data_model/models/_base_unverified.py +0 -0
- /cognite/neat/{core → v0/core}/_data_model/models/conceptual/__init__.py +0 -0
- /cognite/neat/{core → v0/core}/_data_model/models/entities/_constants.py +0 -0
- /cognite/neat/{core → v0/core}/_data_model/models/entities/_wrapped.py +0 -0
- /cognite/neat/{core → v0/core}/_data_model/models/mapping/__init__.py +0 -0
- /cognite/neat/{core → v0/core}/_data_model/models/mapping/_classic2core.yaml +0 -0
- /cognite/neat/{core → v0/core}/_instances/_shared.py +0 -0
- /cognite/neat/{core → v0/core}/_instances/_tracking/__init__.py +0 -0
- /cognite/neat/{core → v0/core}/_instances/examples/Knowledge-Graph-Nordic44-dirty.xml +0 -0
- /cognite/neat/{core → v0/core}/_instances/examples/Knowledge-Graph-Nordic44.xml +0 -0
- /cognite/neat/{core → v0/core}/_instances/examples/__init__.py +0 -0
- /cognite/neat/{core → v0/core}/_instances/examples/skos-capturing-sheet-wind-topics.xlsx +0 -0
- /cognite/neat/{core → v0/core}/_instances/extractors/_classic_cdf/_assets.py +0 -0
- /cognite/neat/{core → v0/core}/_instances/extractors/_classic_cdf/_data_sets.py +0 -0
- /cognite/neat/{core → v0/core}/_instances/extractors/_classic_cdf/_events.py +0 -0
- /cognite/neat/{core → v0/core}/_instances/extractors/_classic_cdf/_files.py +0 -0
- /cognite/neat/{core → v0/core}/_instances/extractors/_classic_cdf/_labels.py +0 -0
- /cognite/neat/{core → v0/core}/_instances/extractors/_classic_cdf/_timeseries.py +0 -0
- /cognite/neat/{core → v0/core}/_instances/loaders/__init__.py +0 -0
- /cognite/neat/{core → v0/core}/_instances/queries/__init__.py +0 -0
- /cognite/neat/{core → v0/core}/_instances/queries/_base.py +0 -0
- /cognite/neat/{core → v0/core}/_instances/queries/_queries.py +0 -0
- /cognite/neat/{core → v0/core}/_instances/transformers/__init__.py +0 -0
- /cognite/neat/{core → v0/core}/_issues/__init__.py +0 -0
- /cognite/neat/{core → v0/core}/_issues/formatters.py +0 -0
- /cognite/neat/{core → v0/core}/_shared.py +0 -0
- /cognite/neat/{core → v0/core}/_store/__init__.py +0 -0
- /cognite/neat/{core → v0/core}/_utils/io_.py +0 -0
- /cognite/neat/{core → v0/core}/_utils/reader/__init__.py +0 -0
- /cognite/neat/{core → v0/core}/_utils/time_.py +0 -0
- /cognite/neat/{core → v0/core}/_utils/xml_.py +0 -0
- /cognite/neat/{session → v0/session}/__init__.py +0 -0
- /cognite/neat/{session → v0/session}/_experimental.py +0 -0
- /cognite/neat/{session → v0/session}/_state/README.md +0 -0
- /cognite/neat/{session → v0/session}/engine/__init__.py +0 -0
- /cognite/neat/{session → v0/session}/engine/_import.py +0 -0
- /cognite/neat/{session → v0/session}/engine/_interface.py +0 -0
- {cognite_neat-0.123.2.dist-info → cognite_neat-0.127.30.dist-info}/licenses/LICENSE +0 -0
|
@@ -3,23 +3,25 @@ import sys
|
|
|
3
3
|
import warnings
|
|
4
4
|
from dataclasses import dataclass
|
|
5
5
|
from datetime import datetime
|
|
6
|
-
from typing import Any, Literal
|
|
6
|
+
from typing import Any, Literal, overload
|
|
7
7
|
|
|
8
8
|
import pandas as pd
|
|
9
9
|
from cognite.client import data_modeling as dm
|
|
10
10
|
from cognite.client.data_classes.data_modeling import ContainerId, ViewId
|
|
11
11
|
from rdflib import Namespace, URIRef
|
|
12
12
|
|
|
13
|
-
from cognite.neat.core._constants import (
|
|
13
|
+
from cognite.neat.v0.core._constants import (
|
|
14
14
|
DEFAULT_NAMESPACE,
|
|
15
15
|
DMS_DIRECT_RELATION_LIST_DEFAULT_LIMIT,
|
|
16
16
|
)
|
|
17
|
-
from cognite.neat.core._data_model.
|
|
17
|
+
from cognite.neat.v0.core._data_model._constants import SPLIT_ON_COMMA_PATTERN
|
|
18
|
+
from cognite.neat.v0.core._data_model.models._base_unverified import (
|
|
18
19
|
UnverifiedComponent,
|
|
19
20
|
UnverifiedDataModel,
|
|
20
21
|
)
|
|
21
|
-
from cognite.neat.core._data_model.models.data_types import DataType
|
|
22
|
-
from cognite.neat.core._data_model.models.entities import (
|
|
22
|
+
from cognite.neat.v0.core._data_model.models.data_types import DataType
|
|
23
|
+
from cognite.neat.v0.core._data_model.models.entities import (
|
|
24
|
+
ContainerConstraintEntity,
|
|
23
25
|
ContainerEntity,
|
|
24
26
|
ContainerIndexEntity,
|
|
25
27
|
DMSNodeEntity,
|
|
@@ -30,9 +32,9 @@ from cognite.neat.core._data_model.models.entities import (
|
|
|
30
32
|
load_connection,
|
|
31
33
|
load_dms_value_type,
|
|
32
34
|
)
|
|
33
|
-
from cognite.neat.core._data_model.models.entities._wrapped import DMSFilter
|
|
34
|
-
from cognite.neat.core._issues.warnings import DeprecatedWarning
|
|
35
|
-
from cognite.neat.core._utils.rdf_ import uri_display_name
|
|
35
|
+
from cognite.neat.v0.core._data_model.models.entities._wrapped import DMSFilter
|
|
36
|
+
from cognite.neat.v0.core._issues.warnings import DeprecatedWarning
|
|
37
|
+
from cognite.neat.v0.core._utils.rdf_ import uri_display_name
|
|
36
38
|
|
|
37
39
|
from ._verified import (
|
|
38
40
|
_DEFAULT_VERSION,
|
|
@@ -137,7 +139,7 @@ class UnverifiedPhysicalProperty(UnverifiedComponent[PhysicalProperty]):
|
|
|
137
139
|
container: str | None = None
|
|
138
140
|
container_property: str | None = None
|
|
139
141
|
index: str | list[str | ContainerIndexEntity] | ContainerIndexEntity | None = None
|
|
140
|
-
constraint: str | list[str] | None = None
|
|
142
|
+
constraint: str | list[str] | list[ContainerConstraintEntity] | ContainerConstraintEntity | None = None
|
|
141
143
|
neatId: str | URIRef | None = None
|
|
142
144
|
conceptual: str | URIRef | None = None
|
|
143
145
|
|
|
@@ -160,34 +162,44 @@ class UnverifiedPhysicalProperty(UnverifiedComponent[PhysicalProperty]):
|
|
|
160
162
|
|
|
161
163
|
def dump(self, default_space: str, default_version: str) -> dict[str, Any]: # type: ignore[override]
|
|
162
164
|
output = super().dump()
|
|
163
|
-
output["View"] = ViewEntity.load(
|
|
164
|
-
|
|
165
|
-
|
|
165
|
+
output["View"] = ViewEntity.load(
|
|
166
|
+
self.view, space=default_space, version=default_version, return_on_failure=True
|
|
167
|
+
)
|
|
168
|
+
output["Value Type"] = load_dms_value_type(
|
|
169
|
+
self.value_type, default_space, default_version, return_on_failure=True
|
|
170
|
+
)
|
|
171
|
+
output["Connection"] = load_connection(self.connection, default_space, default_version, return_on_failure=True)
|
|
166
172
|
output["Container"] = (
|
|
167
|
-
ContainerEntity.load(self.container, space=default_space, version=default_version)
|
|
173
|
+
ContainerEntity.load(self.container, space=default_space, version=default_version, return_on_failure=True)
|
|
168
174
|
if self.container
|
|
169
175
|
else None
|
|
170
176
|
)
|
|
171
177
|
if isinstance(self.index, ContainerIndexEntity) or (isinstance(self.index, str) and "," not in self.index):
|
|
172
|
-
output["Index"] = [ContainerIndexEntity.load(self.index)]
|
|
173
|
-
elif isinstance(self.index, str)
|
|
178
|
+
output["Index"] = [ContainerIndexEntity.load(self.index, return_on_failure=True)]
|
|
179
|
+
elif isinstance(self.index, str):
|
|
174
180
|
output["Index"] = [
|
|
175
|
-
ContainerIndexEntity.load(index.strip()
|
|
181
|
+
ContainerIndexEntity.load(index.strip(), return_on_failure=True)
|
|
182
|
+
for index in SPLIT_ON_COMMA_PATTERN.split(self.index)
|
|
183
|
+
if index.strip()
|
|
176
184
|
]
|
|
177
185
|
elif isinstance(self.index, list):
|
|
178
186
|
index_list: list[ContainerIndexEntity | PhysicalUnknownEntity] = []
|
|
179
187
|
for index in self.index:
|
|
180
188
|
if isinstance(index, ContainerIndexEntity):
|
|
181
189
|
index_list.append(index)
|
|
182
|
-
elif isinstance(index, str)
|
|
190
|
+
elif isinstance(index, str):
|
|
183
191
|
index_list.extend(
|
|
184
|
-
[
|
|
192
|
+
[
|
|
193
|
+
ContainerIndexEntity.load(idx.strip(), return_on_failure=True)
|
|
194
|
+
for idx in SPLIT_ON_COMMA_PATTERN.split(index)
|
|
195
|
+
if idx.strip()
|
|
196
|
+
]
|
|
185
197
|
)
|
|
186
|
-
elif isinstance(index, str):
|
|
187
|
-
index_list.append(ContainerIndexEntity.load(index.strip()))
|
|
188
198
|
else:
|
|
189
199
|
raise TypeError(f"Unexpected type for index: {type(index)}")
|
|
190
200
|
output["Index"] = index_list
|
|
201
|
+
|
|
202
|
+
output["Constraint"] = _parse_constraints(self.constraint, default_space)
|
|
191
203
|
return output
|
|
192
204
|
|
|
193
205
|
def referenced_view(self, default_space: str, default_version: str) -> ViewEntity:
|
|
@@ -240,7 +252,7 @@ class UnverifiedPhysicalContainer(UnverifiedComponent[PhysicalContainer]):
|
|
|
240
252
|
container: str
|
|
241
253
|
name: str | None = None
|
|
242
254
|
description: str | None = None
|
|
243
|
-
constraint: str | None = None
|
|
255
|
+
constraint: str | list[str] | list[ContainerConstraintEntity] | ContainerConstraintEntity | None = None
|
|
244
256
|
neatId: str | URIRef | None = None
|
|
245
257
|
used_for: Literal["node", "edge", "all"] | None = None
|
|
246
258
|
|
|
@@ -250,23 +262,33 @@ class UnverifiedPhysicalContainer(UnverifiedComponent[PhysicalContainer]):
|
|
|
250
262
|
|
|
251
263
|
def dump(self, default_space: str) -> dict[str, Any]: # type: ignore[override]
|
|
252
264
|
output = super().dump()
|
|
253
|
-
output["Container"] = self.as_entity_id(default_space)
|
|
254
|
-
output["Constraint"] = (
|
|
255
|
-
[ContainerEntity.load(constraint.strip(), space=default_space) for constraint in self.constraint.split(",")]
|
|
256
|
-
if self.constraint
|
|
257
|
-
else None
|
|
258
|
-
)
|
|
265
|
+
output["Container"] = self.as_entity_id(default_space, return_on_failure=True)
|
|
266
|
+
output["Constraint"] = _parse_constraints(self.constraint, default_space)
|
|
259
267
|
return output
|
|
260
268
|
|
|
261
|
-
|
|
262
|
-
|
|
269
|
+
@overload
|
|
270
|
+
def as_entity_id(self, default_space: str, return_on_failure: Literal[False] = False) -> ContainerEntity: ...
|
|
271
|
+
|
|
272
|
+
@overload
|
|
273
|
+
def as_entity_id(self, default_space: str, return_on_failure: Literal[True]) -> ContainerEntity | str: ...
|
|
274
|
+
|
|
275
|
+
def as_entity_id(
|
|
276
|
+
self, default_space: str, return_on_failure: Literal[True, False] = False
|
|
277
|
+
) -> ContainerEntity | str:
|
|
278
|
+
return ContainerEntity.load(
|
|
279
|
+
self.container, strict=True, space=default_space, return_on_failure=return_on_failure
|
|
280
|
+
)
|
|
263
281
|
|
|
264
282
|
@classmethod
|
|
265
283
|
def from_container(cls, container: dm.ContainerApply) -> "UnverifiedPhysicalContainer":
|
|
266
284
|
constraints: list[str] = []
|
|
267
|
-
for
|
|
285
|
+
for constraint_name, constraint_obj in (container.constraints or {}).items():
|
|
268
286
|
if isinstance(constraint_obj, dm.RequiresConstraint):
|
|
269
|
-
|
|
287
|
+
constraint = ContainerConstraintEntity(
|
|
288
|
+
prefix="requires", suffix=constraint_name, require=ContainerEntity.from_id(constraint_obj.require)
|
|
289
|
+
)
|
|
290
|
+
constraints.append(str(constraint))
|
|
291
|
+
|
|
270
292
|
# UniquenessConstraint it handled in the properties
|
|
271
293
|
container_entity = ContainerEntity.from_id(container.as_id())
|
|
272
294
|
return cls(
|
|
@@ -299,19 +321,51 @@ class UnverifiedPhysicalView(UnverifiedComponent[PhysicalView]):
|
|
|
299
321
|
|
|
300
322
|
def dump(self, default_space: str, default_version: str) -> dict[str, Any]: # type: ignore[override]
|
|
301
323
|
output = super().dump()
|
|
302
|
-
output["View"] = self.as_entity_id(default_space, default_version)
|
|
303
|
-
output["Implements"] = self._load_implements(default_space, default_version)
|
|
324
|
+
output["View"] = self.as_entity_id(default_space, default_version, return_on_failure=True)
|
|
325
|
+
output["Implements"] = self._load_implements(default_space, default_version, return_on_failure=True)
|
|
304
326
|
return output
|
|
305
327
|
|
|
306
|
-
|
|
307
|
-
|
|
328
|
+
@overload
|
|
329
|
+
def as_entity_id(
|
|
330
|
+
self, default_space: str, default_version: str, return_on_failure: Literal[False] = False
|
|
331
|
+
) -> ViewEntity: ...
|
|
332
|
+
|
|
333
|
+
@overload
|
|
334
|
+
def as_entity_id(
|
|
335
|
+
self, default_space: str, default_version: str, return_on_failure: Literal[True]
|
|
336
|
+
) -> ViewEntity | str: ...
|
|
337
|
+
|
|
338
|
+
def as_entity_id(
|
|
339
|
+
self, default_space: str, default_version: str, return_on_failure: Literal[True, False] = False
|
|
340
|
+
) -> ViewEntity | str:
|
|
341
|
+
return ViewEntity.load(
|
|
342
|
+
self.view, strict=True, space=default_space, version=default_version, return_on_failure=return_on_failure
|
|
343
|
+
)
|
|
308
344
|
|
|
309
|
-
|
|
345
|
+
@overload
|
|
346
|
+
def _load_implements(
|
|
347
|
+
self, default_space: str, default_version: str, return_on_failure: Literal[False] = False
|
|
348
|
+
) -> list[ViewEntity] | None: ...
|
|
349
|
+
|
|
350
|
+
@overload
|
|
351
|
+
def _load_implements(
|
|
352
|
+
self, default_space: str, default_version: str, return_on_failure: Literal[True]
|
|
353
|
+
) -> list[ViewEntity | str] | None: ...
|
|
354
|
+
|
|
355
|
+
def _load_implements(
|
|
356
|
+
self, default_space: str, default_version: str, return_on_failure: Literal[True, False] = False
|
|
357
|
+
) -> list[ViewEntity] | list[ViewEntity | str] | None:
|
|
310
358
|
self.implements = self.implements.strip() if self.implements else None
|
|
311
359
|
|
|
312
360
|
return (
|
|
313
361
|
[
|
|
314
|
-
ViewEntity.load(
|
|
362
|
+
ViewEntity.load(
|
|
363
|
+
implement,
|
|
364
|
+
strict=True,
|
|
365
|
+
space=default_space,
|
|
366
|
+
version=default_version,
|
|
367
|
+
return_on_failure=return_on_failure,
|
|
368
|
+
)
|
|
315
369
|
for implement in self.implements.split(",")
|
|
316
370
|
]
|
|
317
371
|
if self.implements
|
|
@@ -354,7 +408,7 @@ class UnverifiedPhysicalNodeType(UnverifiedComponent[PhysicalNodeType]):
|
|
|
354
408
|
|
|
355
409
|
def dump(self, default_space: str, **_) -> dict[str, Any]: # type: ignore
|
|
356
410
|
output = super().dump()
|
|
357
|
-
output["Node"] = DMSNodeEntity.load(self.node, space=default_space)
|
|
411
|
+
output["Node"] = DMSNodeEntity.load(self.node, space=default_space, return_on_failure=True)
|
|
358
412
|
return output
|
|
359
413
|
|
|
360
414
|
|
|
@@ -452,3 +506,52 @@ class UnverifiedPhysicalDataModel(UnverifiedDataModel[PhysicalDataModel]):
|
|
|
452
506
|
def imported_views_and_containers_ids(self) -> tuple[set[ViewId], set[ContainerId]]:
|
|
453
507
|
views, containers = self.imported_views_and_containers()
|
|
454
508
|
return {view.as_id() for view in views}, {container.as_id() for container in containers}
|
|
509
|
+
|
|
510
|
+
|
|
511
|
+
def _parse_constraints(
|
|
512
|
+
constraint: str | list[str] | list[ContainerConstraintEntity] | ContainerConstraintEntity | None,
|
|
513
|
+
default_space: str | None = None,
|
|
514
|
+
) -> list[ContainerConstraintEntity | PhysicalUnknownEntity] | None:
|
|
515
|
+
"""Parse constraint input into a standardized list of ContainerConstraintEntity objects.
|
|
516
|
+
|
|
517
|
+
Args:
|
|
518
|
+
constraint: The constraint input in various formats
|
|
519
|
+
default_space: Default space to use when loading constraint entities
|
|
520
|
+
|
|
521
|
+
Returns:
|
|
522
|
+
List of parsed constraint entities, or None if no constraints
|
|
523
|
+
"""
|
|
524
|
+
if constraint is None:
|
|
525
|
+
return None
|
|
526
|
+
|
|
527
|
+
if isinstance(constraint, ContainerConstraintEntity):
|
|
528
|
+
return [constraint]
|
|
529
|
+
|
|
530
|
+
if isinstance(constraint, str) and "," not in constraint:
|
|
531
|
+
return [ContainerConstraintEntity.load(constraint, return_on_failure=True, space=default_space)]
|
|
532
|
+
|
|
533
|
+
if isinstance(constraint, str):
|
|
534
|
+
return [
|
|
535
|
+
ContainerConstraintEntity.load(constraint_item.strip(), return_on_failure=True, space=default_space)
|
|
536
|
+
for constraint_item in SPLIT_ON_COMMA_PATTERN.split(constraint)
|
|
537
|
+
if constraint_item.strip()
|
|
538
|
+
]
|
|
539
|
+
|
|
540
|
+
if isinstance(constraint, list):
|
|
541
|
+
constraint_list: list[ContainerConstraintEntity | PhysicalUnknownEntity] = []
|
|
542
|
+
for constraint_item in constraint:
|
|
543
|
+
if isinstance(constraint_item, ContainerConstraintEntity):
|
|
544
|
+
constraint_list.append(constraint_item)
|
|
545
|
+
elif isinstance(constraint_item, str):
|
|
546
|
+
constraint_list.extend(
|
|
547
|
+
[
|
|
548
|
+
ContainerConstraintEntity.load(idx.strip(), return_on_failure=True, space=default_space)
|
|
549
|
+
for idx in SPLIT_ON_COMMA_PATTERN.split(constraint_item)
|
|
550
|
+
if idx.strip()
|
|
551
|
+
]
|
|
552
|
+
)
|
|
553
|
+
else:
|
|
554
|
+
raise TypeError(f"Unexpected type for constraint: {type(constraint_item)}")
|
|
555
|
+
return constraint_list
|
|
556
|
+
|
|
557
|
+
raise TypeError(f"Unexpected type for constraint: {type(constraint)}")
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import warnings
|
|
2
2
|
from collections import Counter, defaultdict
|
|
3
3
|
from collections.abc import Mapping
|
|
4
|
+
from dataclasses import dataclass
|
|
4
5
|
from functools import lru_cache
|
|
6
|
+
from typing import cast
|
|
5
7
|
|
|
6
8
|
from cognite.client import data_modeling as dm
|
|
7
9
|
from cognite.client.data_classes.data_modeling import ContainerList, ViewId, ViewList
|
|
@@ -12,49 +14,68 @@ from cognite.client.data_classes.data_modeling.views import (
|
|
|
12
14
|
ViewPropertyApply,
|
|
13
15
|
)
|
|
14
16
|
|
|
15
|
-
from cognite.neat.core._client import NeatClient
|
|
16
|
-
from cognite.neat.core._client.data_classes.data_modeling import ViewApplyDict
|
|
17
|
-
from cognite.neat.core._client.data_classes.schema import DMSSchema
|
|
18
|
-
from cognite.neat.core._constants import (
|
|
17
|
+
from cognite.neat.v0.core._client import NeatClient
|
|
18
|
+
from cognite.neat.v0.core._client.data_classes.data_modeling import ViewApplyDict
|
|
19
|
+
from cognite.neat.v0.core._client.data_classes.schema import DMSSchema
|
|
20
|
+
from cognite.neat.v0.core._constants import (
|
|
19
21
|
COGNITE_MODELS,
|
|
20
22
|
COGNITE_SPACES,
|
|
21
23
|
DMS_CONTAINER_PROPERTY_SIZE_LIMIT,
|
|
22
24
|
DMS_VIEW_CONTAINER_SIZE_LIMIT,
|
|
25
|
+
get_base_concepts,
|
|
23
26
|
)
|
|
24
|
-
from cognite.neat.core._data_model.models.
|
|
25
|
-
from cognite.neat.core._data_model.models.
|
|
26
|
-
from cognite.neat.core._data_model.models.entities
|
|
27
|
+
from cognite.neat.v0.core._data_model.models._import_contexts import ImportContext, SpreadsheetContext
|
|
28
|
+
from cognite.neat.v0.core._data_model.models.data_types import DataType
|
|
29
|
+
from cognite.neat.v0.core._data_model.models.entities import ContainerEntity, RawFilter
|
|
30
|
+
from cognite.neat.v0.core._data_model.models.entities._single_value import (
|
|
31
|
+
ContainerIndexEntity,
|
|
27
32
|
ViewEntity,
|
|
28
33
|
)
|
|
29
|
-
from cognite.neat.core._issues import IssueList, NeatError
|
|
30
|
-
from cognite.neat.core._issues.errors import (
|
|
34
|
+
from cognite.neat.v0.core._issues import IssueList, NeatError
|
|
35
|
+
from cognite.neat.v0.core._issues.errors import (
|
|
31
36
|
CDFMissingClientError,
|
|
32
37
|
PropertyDefinitionDuplicatedError,
|
|
38
|
+
PropertyInvalidDefinitionError,
|
|
33
39
|
PropertyMappingDuplicatedError,
|
|
34
40
|
PropertyNotFoundError,
|
|
35
41
|
ResourceDuplicatedError,
|
|
36
42
|
ResourceNotFoundError,
|
|
37
43
|
ReversedConnectionNotFeasibleError,
|
|
38
44
|
)
|
|
39
|
-
from cognite.neat.core._issues.errors._external import CDFMissingResourcesError
|
|
40
|
-
from cognite.neat.core._issues.warnings import (
|
|
45
|
+
from cognite.neat.v0.core._issues.errors._external import CDFMissingResourcesError
|
|
46
|
+
from cognite.neat.v0.core._issues.warnings import (
|
|
41
47
|
NotSupportedHasDataFilterLimitWarning,
|
|
42
48
|
NotSupportedViewContainerLimitWarning,
|
|
43
49
|
ReversedConnectionNotFeasibleWarning,
|
|
44
50
|
UndefinedViewWarning,
|
|
45
51
|
user_modeling,
|
|
46
52
|
)
|
|
47
|
-
from cognite.neat.core._issues.warnings.
|
|
53
|
+
from cognite.neat.v0.core._issues.warnings._models import ViewWithoutPropertiesWarning
|
|
54
|
+
from cognite.neat.v0.core._issues.warnings.user_modeling import (
|
|
48
55
|
ContainerPropertyLimitWarning,
|
|
49
56
|
DirectRelationMissingSourceWarning,
|
|
50
57
|
NotNeatSupportedFilterWarning,
|
|
51
58
|
)
|
|
52
|
-
from cognite.neat.core._utils.
|
|
53
|
-
from cognite.neat.core._utils.text import humanize_collection
|
|
59
|
+
from cognite.neat.v0.core._utils.text import humanize_collection
|
|
54
60
|
|
|
55
61
|
from ._verified import PhysicalDataModel, PhysicalProperty
|
|
56
62
|
|
|
57
63
|
|
|
64
|
+
@dataclass
|
|
65
|
+
class _ContainerPropertyIndex:
|
|
66
|
+
"""This is a helper class used in the indices validation
|
|
67
|
+
|
|
68
|
+
Args:
|
|
69
|
+
location: The index of the property in the properties list.
|
|
70
|
+
property_: The physical property associated with the container.
|
|
71
|
+
index: The index entity that defines the container property.
|
|
72
|
+
"""
|
|
73
|
+
|
|
74
|
+
location: int
|
|
75
|
+
property_: PhysicalProperty
|
|
76
|
+
index: ContainerIndexEntity
|
|
77
|
+
|
|
78
|
+
|
|
58
79
|
class PhysicalValidation:
|
|
59
80
|
"""This class does all the validation of the physical data model that
|
|
60
81
|
have dependencies between components."""
|
|
@@ -63,15 +84,23 @@ class PhysicalValidation:
|
|
|
63
84
|
self,
|
|
64
85
|
data_model: PhysicalDataModel,
|
|
65
86
|
client: NeatClient | None = None,
|
|
66
|
-
|
|
87
|
+
context: ImportContext | None = None,
|
|
67
88
|
) -> None:
|
|
89
|
+
# import here to avoid circular import issues
|
|
90
|
+
from cognite.neat.v0.core._data_model.analysis._base import DataModelAnalysis
|
|
91
|
+
|
|
68
92
|
self._data_model = data_model
|
|
69
93
|
self._client = client
|
|
70
94
|
self._metadata = data_model.metadata
|
|
71
95
|
self._properties = data_model.properties
|
|
72
96
|
self._containers = data_model.containers
|
|
73
97
|
self._views = data_model.views
|
|
74
|
-
self._read_info_by_spreadsheet =
|
|
98
|
+
self._read_info_by_spreadsheet = context if isinstance(context, SpreadsheetContext) else SpreadsheetContext()
|
|
99
|
+
|
|
100
|
+
self.analysis = DataModelAnalysis(physical=self._data_model)
|
|
101
|
+
self._cdf_concepts = {
|
|
102
|
+
ViewEntity.load(concept_as_string) for concept_as_string in get_base_concepts(base_model="CogniteCore")
|
|
103
|
+
}
|
|
75
104
|
|
|
76
105
|
def imported_views_and_containers_ids(
|
|
77
106
|
self, include_views_with_no_properties: bool = True
|
|
@@ -93,9 +122,9 @@ class PhysicalValidation:
|
|
|
93
122
|
view_with_properties.add(prop.view)
|
|
94
123
|
|
|
95
124
|
for container in self._containers or []:
|
|
96
|
-
for
|
|
97
|
-
if
|
|
98
|
-
imported_containers.add(
|
|
125
|
+
for constraint in container.constraint or []:
|
|
126
|
+
if constraint.require not in existing_containers:
|
|
127
|
+
imported_containers.add(cast(ContainerEntity, constraint.require))
|
|
99
128
|
|
|
100
129
|
if include_views_with_no_properties:
|
|
101
130
|
extra_views = existing_views - view_with_properties
|
|
@@ -145,16 +174,21 @@ class PhysicalValidation:
|
|
|
145
174
|
all_properties_by_ids = {**ref_properties_by_ids, **properties_by_ids}
|
|
146
175
|
view_properties_by_id = self._as_view_properties_by_id(properties_by_ids)
|
|
147
176
|
parents_view_ids_by_child_id = self._parent_view_ids_by_child_id(all_views_by_id)
|
|
177
|
+
container_properties_by_id = self._create_container_properties_by_id()
|
|
148
178
|
|
|
149
179
|
issue_list = IssueList()
|
|
150
180
|
|
|
151
181
|
# Validated for duplicated resource
|
|
152
182
|
issue_list.extend(self._duplicated_resources())
|
|
153
183
|
|
|
184
|
+
# Validate if views are defined (i.e. have at least one property defined, or inherited)
|
|
185
|
+
issue_list.extend(self._views_without_properties_exist())
|
|
186
|
+
|
|
154
187
|
# Neat DMS classes Validation
|
|
155
188
|
# These are errors that can only happen due to the format of the Neat DMS classes
|
|
156
189
|
issue_list.extend(self._validate_raw_filter())
|
|
157
|
-
issue_list.extend(self._consistent_container_properties())
|
|
190
|
+
issue_list.extend(self._consistent_container_properties(container_properties_by_id))
|
|
191
|
+
issue_list.extend(self._valid_composite_container_indices(container_properties_by_id))
|
|
158
192
|
issue_list.extend(self._validate_value_type_existence())
|
|
159
193
|
issue_list.extend(
|
|
160
194
|
self._validate_property_referenced_views_and_containers_exists(all_views_by_id, all_containers_by_id)
|
|
@@ -172,6 +206,23 @@ class PhysicalValidation:
|
|
|
172
206
|
issue_list.extend(self._same_space_views_and_data_model())
|
|
173
207
|
return issue_list
|
|
174
208
|
|
|
209
|
+
def _views_without_properties_exist(self) -> IssueList:
|
|
210
|
+
"""Check if there are views that do not have any properties defined directly or inherited."""
|
|
211
|
+
issue_list = IssueList()
|
|
212
|
+
views = {view.view for view in self._views}
|
|
213
|
+
ancestors_by_view = self.analysis.implements_by_view(include_ancestors=True, include_different_space=True)
|
|
214
|
+
views_with_properties = self.analysis.defined_views().union(self._cdf_concepts)
|
|
215
|
+
|
|
216
|
+
if candidate_views := views.difference(views_with_properties):
|
|
217
|
+
for view in candidate_views:
|
|
218
|
+
# Here we check if at least one of the ancestors of the view has properties
|
|
219
|
+
if (ancestors := ancestors_by_view.get(view)) and ancestors.intersection(views_with_properties):
|
|
220
|
+
continue
|
|
221
|
+
|
|
222
|
+
issue_list.append_if_not_exist(ViewWithoutPropertiesWarning(view_id=view.as_id()))
|
|
223
|
+
|
|
224
|
+
return issue_list
|
|
225
|
+
|
|
175
226
|
def _same_space_views_and_data_model(self) -> IssueList:
|
|
176
227
|
issue_list = IssueList()
|
|
177
228
|
|
|
@@ -320,13 +371,27 @@ class PhysicalValidation:
|
|
|
320
371
|
parents_by_view[view_id] = get_parents(view_id)
|
|
321
372
|
return parents_by_view
|
|
322
373
|
|
|
323
|
-
def
|
|
374
|
+
def _create_container_properties_by_id(
|
|
375
|
+
self,
|
|
376
|
+
) -> dict[tuple[ContainerEntity, str], list[tuple[int, PhysicalProperty]]]:
|
|
377
|
+
"""Create a mapping of container properties with their location in the properties list.
|
|
378
|
+
|
|
379
|
+
Returns:
|
|
380
|
+
dict[tuple[ContainerEntity, str], list[tuple[int, PhysicalProperty]]]: A dictionary where the key is a tuple
|
|
381
|
+
of (ContainerEntity, property name) and the value is a list of tuples of (int, PhysicalProperty) where
|
|
382
|
+
int is the index of the property in the properties list and PhysicalProperty is the property itself.
|
|
383
|
+
"""
|
|
324
384
|
container_properties_by_id: dict[tuple[ContainerEntity, str], list[tuple[int, PhysicalProperty]]] = defaultdict(
|
|
325
385
|
list
|
|
326
386
|
)
|
|
327
387
|
for prop_no, prop in enumerate(self._properties):
|
|
328
388
|
if prop.container and prop.container_property:
|
|
329
389
|
container_properties_by_id[(prop.container, prop.container_property)].append((prop_no, prop))
|
|
390
|
+
return container_properties_by_id
|
|
391
|
+
|
|
392
|
+
def _consistent_container_properties(
|
|
393
|
+
self, container_properties_by_id: dict[tuple[ContainerEntity, str], list[tuple[int, PhysicalProperty]]]
|
|
394
|
+
) -> IssueList:
|
|
330
395
|
properties_sheet = self._read_info_by_spreadsheet.get("Properties")
|
|
331
396
|
errors = IssueList()
|
|
332
397
|
for (container, prop_name), properties in container_properties_by_id.items():
|
|
@@ -392,11 +457,11 @@ class PhysicalValidation:
|
|
|
392
457
|
)
|
|
393
458
|
)
|
|
394
459
|
index_definitions = {
|
|
395
|
-
",".join([index
|
|
460
|
+
",".join([str(index) for index in prop.index]) for _, prop in properties if prop.index is not None
|
|
396
461
|
}
|
|
397
462
|
if len(index_definitions) > 1:
|
|
398
463
|
errors.append(
|
|
399
|
-
PropertyDefinitionDuplicatedError
|
|
464
|
+
PropertyDefinitionDuplicatedError(
|
|
400
465
|
container_id,
|
|
401
466
|
"container",
|
|
402
467
|
prop_name,
|
|
@@ -406,8 +471,11 @@ class PhysicalValidation:
|
|
|
406
471
|
)
|
|
407
472
|
)
|
|
408
473
|
constraint_definitions = {
|
|
409
|
-
",".join(
|
|
474
|
+
",".join([str(constraint) for constraint in prop.constraint])
|
|
475
|
+
for _, prop in properties
|
|
476
|
+
if prop.constraint is not None
|
|
410
477
|
}
|
|
478
|
+
|
|
411
479
|
if len(constraint_definitions) > 1:
|
|
412
480
|
errors.append(
|
|
413
481
|
PropertyDefinitionDuplicatedError[dm.ContainerId](
|
|
@@ -422,6 +490,104 @@ class PhysicalValidation:
|
|
|
422
490
|
|
|
423
491
|
return errors
|
|
424
492
|
|
|
493
|
+
def _valid_composite_container_indices(
|
|
494
|
+
self, container_properties_by_id: dict[tuple[ContainerEntity, str], list[tuple[int, PhysicalProperty]]]
|
|
495
|
+
) -> IssueList:
|
|
496
|
+
"""Validate that the indices on the container properties are valid."""
|
|
497
|
+
index_properties_by_container_index: dict[tuple[ContainerEntity, str], list[_ContainerPropertyIndex]] = (
|
|
498
|
+
defaultdict(list)
|
|
499
|
+
)
|
|
500
|
+
for (container, _), properties in container_properties_by_id.items():
|
|
501
|
+
for row_no, prop in properties:
|
|
502
|
+
for index in prop.index or []:
|
|
503
|
+
index_properties_by_container_index[(container, index.suffix)].append(
|
|
504
|
+
_ContainerPropertyIndex(row_no, prop, index)
|
|
505
|
+
)
|
|
506
|
+
|
|
507
|
+
properties_sheet_info = self._read_info_by_spreadsheet.get("Properties")
|
|
508
|
+
errors = IssueList()
|
|
509
|
+
for (container, _), index_properties in index_properties_by_container_index.items():
|
|
510
|
+
if len(index_properties) <= 1:
|
|
511
|
+
# If there is only one property in the index, this is already validated in the field_validator
|
|
512
|
+
# of the PhysicalProperty class. This validation is only for composite indices.
|
|
513
|
+
continue
|
|
514
|
+
container_id = container.as_id()
|
|
515
|
+
row_numbers = tuple([index_prop.location for index_prop in index_properties])
|
|
516
|
+
if properties_sheet_info:
|
|
517
|
+
row_numbers = tuple([properties_sheet_info.adjusted_row_number(row_no) for row_no in row_numbers])
|
|
518
|
+
|
|
519
|
+
if order_missing_error := self._validate_container_indices_has_order(
|
|
520
|
+
index_properties, row_numbers, container_id
|
|
521
|
+
):
|
|
522
|
+
errors.append(order_missing_error)
|
|
523
|
+
same_order_errors = self._validate_container_indices_same_order(index_properties, row_numbers, container_id)
|
|
524
|
+
errors.extend(same_order_errors)
|
|
525
|
+
return errors
|
|
526
|
+
|
|
527
|
+
@staticmethod
|
|
528
|
+
def _validate_container_indices_has_order(
|
|
529
|
+
index_properties: list[_ContainerPropertyIndex], row_numbers: tuple[int, ...], container_id: dm.ContainerId
|
|
530
|
+
) -> PropertyInvalidDefinitionError | None:
|
|
531
|
+
property_names: list[str] = []
|
|
532
|
+
indices: list[ContainerIndexEntity] = []
|
|
533
|
+
for prop_index in index_properties:
|
|
534
|
+
if prop_index.index.order is None:
|
|
535
|
+
property_names.append(prop_index.property_.view_property)
|
|
536
|
+
indices.append(prop_index.index)
|
|
537
|
+
|
|
538
|
+
if not property_names:
|
|
539
|
+
return None
|
|
540
|
+
|
|
541
|
+
if len(set(property_names)) == 1:
|
|
542
|
+
# If this is the same property, this is not a composite index, but a poorly defined single property
|
|
543
|
+
# index. This will be caught by the PropertyDefinitionDuplicatedError.
|
|
544
|
+
return None
|
|
545
|
+
|
|
546
|
+
properties_str = humanize_collection(property_names)
|
|
547
|
+
fixed_indices = [str(index.model_copy(update={"order": no})) for no, index in enumerate(indices, 1)]
|
|
548
|
+
message = (
|
|
549
|
+
"You must specify the order when using a composite index. "
|
|
550
|
+
f"For example {humanize_collection(fixed_indices)}."
|
|
551
|
+
)
|
|
552
|
+
return PropertyInvalidDefinitionError(
|
|
553
|
+
container_id,
|
|
554
|
+
"container",
|
|
555
|
+
properties_str,
|
|
556
|
+
message,
|
|
557
|
+
row_numbers,
|
|
558
|
+
"rows",
|
|
559
|
+
)
|
|
560
|
+
|
|
561
|
+
@staticmethod
|
|
562
|
+
def _validate_container_indices_same_order(
|
|
563
|
+
index_properties: list[_ContainerPropertyIndex], row_numbers: tuple[int, ...], container_id: dm.ContainerId
|
|
564
|
+
) -> list[PropertyInvalidDefinitionError]:
|
|
565
|
+
"""Checks whether there are multiple properties with the same order in a composite index."""
|
|
566
|
+
properties_by_order: dict[int, list[tuple[int, PhysicalProperty]]] = defaultdict(list)
|
|
567
|
+
for index_prop in index_properties:
|
|
568
|
+
if index_prop.index.order is not None:
|
|
569
|
+
properties_by_order[index_prop.index.order].append((index_prop.location, index_prop.property_))
|
|
570
|
+
same_order_errors: list[PropertyInvalidDefinitionError] = []
|
|
571
|
+
for order, props in properties_by_order.items():
|
|
572
|
+
if len(props) > 1:
|
|
573
|
+
properties_str = humanize_collection([prop.view_property for _, prop in props])
|
|
574
|
+
message = (
|
|
575
|
+
"You cannot have multiple properties with the same order in a composite index. "
|
|
576
|
+
f"Got order={order} for all composite properties."
|
|
577
|
+
"Please ensure that each property has an unique order."
|
|
578
|
+
)
|
|
579
|
+
same_order_errors.append(
|
|
580
|
+
PropertyInvalidDefinitionError(
|
|
581
|
+
container_id,
|
|
582
|
+
"container",
|
|
583
|
+
properties_str,
|
|
584
|
+
message,
|
|
585
|
+
row_numbers,
|
|
586
|
+
"rows",
|
|
587
|
+
)
|
|
588
|
+
)
|
|
589
|
+
return same_order_errors
|
|
590
|
+
|
|
425
591
|
@staticmethod
|
|
426
592
|
def _containers_are_proper_size(dms_schema: DMSSchema) -> IssueList:
|
|
427
593
|
errors = IssueList()
|