cognite-neat 0.123.26__py3-none-any.whl → 1.0.22__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cognite/neat/__init__.py +4 -3
- cognite/neat/_client/__init__.py +5 -0
- cognite/neat/_client/api.py +8 -0
- cognite/neat/_client/client.py +21 -0
- cognite/neat/_client/config.py +40 -0
- cognite/neat/_client/containers_api.py +138 -0
- cognite/neat/_client/data_classes.py +44 -0
- cognite/neat/_client/data_model_api.py +115 -0
- cognite/neat/_client/init/credentials.py +70 -0
- cognite/neat/_client/init/env_vars.py +131 -0
- cognite/neat/_client/init/main.py +51 -0
- cognite/neat/_client/spaces_api.py +115 -0
- cognite/neat/_client/statistics_api.py +24 -0
- cognite/neat/_client/views_api.py +144 -0
- cognite/neat/_config.py +266 -0
- cognite/neat/_data_model/_analysis.py +571 -0
- cognite/neat/_data_model/_constants.py +74 -0
- cognite/neat/_data_model/_identifiers.py +61 -0
- cognite/neat/_data_model/_shared.py +41 -0
- cognite/neat/_data_model/_snapshot.py +134 -0
- cognite/neat/_data_model/deployer/_differ.py +140 -0
- cognite/neat/_data_model/deployer/_differ_container.py +360 -0
- cognite/neat/_data_model/deployer/_differ_data_model.py +54 -0
- cognite/neat/_data_model/deployer/_differ_space.py +9 -0
- cognite/neat/_data_model/deployer/_differ_view.py +299 -0
- cognite/neat/_data_model/deployer/data_classes.py +644 -0
- cognite/neat/_data_model/deployer/deployer.py +431 -0
- cognite/neat/_data_model/exporters/__init__.py +15 -0
- cognite/neat/_data_model/exporters/_api_exporter.py +37 -0
- cognite/neat/_data_model/exporters/_base.py +24 -0
- cognite/neat/_data_model/exporters/_table_exporter/exporter.py +128 -0
- cognite/neat/_data_model/exporters/_table_exporter/workbook.py +409 -0
- cognite/neat/_data_model/exporters/_table_exporter/writer.py +480 -0
- cognite/neat/_data_model/importers/__init__.py +5 -0
- cognite/neat/_data_model/importers/_api_importer.py +166 -0
- cognite/neat/_data_model/importers/_base.py +16 -0
- cognite/neat/_data_model/importers/_table_importer/data_classes.py +344 -0
- cognite/neat/_data_model/importers/_table_importer/importer.py +192 -0
- cognite/neat/_data_model/importers/_table_importer/reader.py +1102 -0
- cognite/neat/_data_model/importers/_table_importer/source.py +94 -0
- cognite/neat/_data_model/models/conceptual/_base.py +18 -0
- cognite/neat/_data_model/models/conceptual/_concept.py +67 -0
- cognite/neat/_data_model/models/conceptual/_data_model.py +51 -0
- cognite/neat/_data_model/models/conceptual/_properties.py +104 -0
- cognite/neat/_data_model/models/conceptual/_property.py +105 -0
- cognite/neat/_data_model/models/dms/__init__.py +206 -0
- cognite/neat/_data_model/models/dms/_base.py +31 -0
- cognite/neat/_data_model/models/dms/_constants.py +48 -0
- cognite/neat/_data_model/models/dms/_constraints.py +42 -0
- cognite/neat/_data_model/models/dms/_container.py +159 -0
- cognite/neat/_data_model/models/dms/_data_model.py +95 -0
- cognite/neat/_data_model/models/dms/_data_types.py +195 -0
- cognite/neat/_data_model/models/dms/_http.py +28 -0
- cognite/neat/_data_model/models/dms/_indexes.py +30 -0
- cognite/neat/_data_model/models/dms/_limits.py +96 -0
- cognite/neat/_data_model/models/dms/_references.py +141 -0
- cognite/neat/_data_model/models/dms/_schema.py +18 -0
- cognite/neat/_data_model/models/dms/_space.py +48 -0
- cognite/neat/_data_model/models/dms/_types.py +17 -0
- cognite/neat/_data_model/models/dms/_view_filter.py +310 -0
- cognite/neat/_data_model/models/dms/_view_property.py +235 -0
- cognite/neat/_data_model/models/dms/_views.py +216 -0
- cognite/neat/_data_model/models/entities/__init__.py +50 -0
- cognite/neat/_data_model/models/entities/_base.py +101 -0
- cognite/neat/_data_model/models/entities/_constants.py +22 -0
- cognite/neat/_data_model/models/entities/_data_types.py +144 -0
- cognite/neat/_data_model/models/entities/_identifiers.py +61 -0
- cognite/neat/_data_model/models/entities/_parser.py +226 -0
- cognite/neat/_data_model/validation/dms/__init__.py +75 -0
- cognite/neat/_data_model/validation/dms/_ai_readiness.py +381 -0
- cognite/neat/_data_model/validation/dms/_base.py +25 -0
- cognite/neat/_data_model/validation/dms/_connections.py +681 -0
- cognite/neat/_data_model/validation/dms/_consistency.py +58 -0
- cognite/neat/_data_model/validation/dms/_containers.py +199 -0
- cognite/neat/_data_model/validation/dms/_limits.py +368 -0
- cognite/neat/_data_model/validation/dms/_orchestrator.py +70 -0
- cognite/neat/_data_model/validation/dms/_views.py +164 -0
- cognite/neat/_exceptions.py +68 -0
- cognite/neat/_issues.py +68 -0
- cognite/neat/_session/__init__.py +3 -0
- cognite/neat/_session/_html/_render.py +30 -0
- cognite/neat/_session/_html/static/__init__.py +8 -0
- cognite/neat/_session/_html/static/deployment.css +476 -0
- cognite/neat/_session/_html/static/deployment.js +181 -0
- cognite/neat/_session/_html/static/issues.css +211 -0
- cognite/neat/_session/_html/static/issues.js +168 -0
- cognite/neat/_session/_html/static/shared.css +186 -0
- cognite/neat/_session/_html/templates/__init__.py +4 -0
- cognite/neat/_session/_html/templates/deployment.html +80 -0
- cognite/neat/_session/_html/templates/issues.html +45 -0
- cognite/neat/_session/_issues.py +81 -0
- cognite/neat/_session/_physical.py +294 -0
- cognite/neat/_session/_result/__init__.py +3 -0
- cognite/neat/_session/_result/_deployment/__init__.py +0 -0
- cognite/neat/_session/_result/_deployment/_physical/__init__.py +0 -0
- cognite/neat/_session/_result/_deployment/_physical/_changes.py +196 -0
- cognite/neat/_session/_result/_deployment/_physical/_statistics.py +180 -0
- cognite/neat/_session/_result/_deployment/_physical/serializer.py +35 -0
- cognite/neat/_session/_result/_result.py +31 -0
- cognite/neat/_session/_session.py +81 -0
- cognite/neat/_session/_usage_analytics/__init__.py +0 -0
- cognite/neat/_session/_usage_analytics/_collector.py +131 -0
- cognite/neat/_session/_usage_analytics/_constants.py +23 -0
- cognite/neat/_session/_usage_analytics/_storage.py +240 -0
- cognite/neat/_session/_wrappers.py +101 -0
- cognite/neat/_state_machine/__init__.py +10 -0
- cognite/neat/_state_machine/_base.py +37 -0
- cognite/neat/_state_machine/_states.py +52 -0
- cognite/neat/_store/__init__.py +3 -0
- cognite/neat/_store/_provenance.py +88 -0
- cognite/neat/_store/_store.py +220 -0
- cognite/neat/_utils/__init__.py +0 -0
- cognite/neat/_utils/_reader.py +194 -0
- cognite/neat/_utils/auxiliary.py +49 -0
- cognite/neat/_utils/collection.py +11 -0
- cognite/neat/_utils/http_client/__init__.py +39 -0
- cognite/neat/_utils/http_client/_client.py +245 -0
- cognite/neat/_utils/http_client/_config.py +19 -0
- cognite/neat/_utils/http_client/_data_classes.py +294 -0
- cognite/neat/_utils/http_client/_tracker.py +31 -0
- cognite/neat/_utils/repo.py +19 -0
- cognite/neat/_utils/text.py +71 -0
- cognite/neat/_utils/useful_types.py +37 -0
- cognite/neat/_utils/validation.py +154 -0
- cognite/neat/_v0/__init__.py +0 -0
- cognite/neat/_v0/core/__init__.py +0 -0
- cognite/neat/_v0/core/_client/_api/__init__.py +0 -0
- cognite/neat/{core → _v0/core}/_client/_api/data_modeling_loaders.py +8 -7
- cognite/neat/{core → _v0/core}/_client/_api/neat_instances.py +5 -5
- cognite/neat/{core → _v0/core}/_client/_api/schema.py +5 -5
- cognite/neat/{core → _v0/core}/_client/_api/statistics.py +3 -3
- cognite/neat/{core → _v0/core}/_client/_api_client.py +1 -1
- cognite/neat/_v0/core/_client/data_classes/__init__.py +0 -0
- cognite/neat/{core → _v0/core}/_client/data_classes/schema.py +4 -4
- cognite/neat/{core → _v0/core}/_client/testing.py +1 -1
- cognite/neat/{core → _v0/core}/_constants.py +5 -3
- cognite/neat/_v0/core/_data_model/__init__.py +0 -0
- cognite/neat/{core → _v0/core}/_data_model/_constants.py +7 -0
- cognite/neat/{core → _v0/core}/_data_model/_shared.py +4 -4
- cognite/neat/{core → _v0/core}/_data_model/analysis/_base.py +8 -8
- cognite/neat/{core → _v0/core}/_data_model/exporters/__init__.py +1 -2
- cognite/neat/{core → _v0/core}/_data_model/exporters/_base.py +7 -7
- cognite/neat/{core → _v0/core}/_data_model/exporters/_data_model2dms.py +9 -9
- cognite/neat/{core → _v0/core}/_data_model/exporters/_data_model2excel.py +12 -12
- cognite/neat/{core → _v0/core}/_data_model/exporters/_data_model2instance_template.py +4 -4
- cognite/neat/{core/_data_model/exporters/_data_model2ontology.py → _v0/core/_data_model/exporters/_data_model2semantic_model.py} +126 -116
- cognite/neat/{core → _v0/core}/_data_model/exporters/_data_model2yaml.py +1 -1
- cognite/neat/{core → _v0/core}/_data_model/importers/_base.py +5 -5
- cognite/neat/{core → _v0/core}/_data_model/importers/_base_file_reader.py +2 -2
- cognite/neat/{core → _v0/core}/_data_model/importers/_dict2data_model.py +5 -5
- cognite/neat/{core → _v0/core}/_data_model/importers/_dms2data_model.py +18 -15
- cognite/neat/{core → _v0/core}/_data_model/importers/_graph2data_model.py +12 -12
- cognite/neat/{core → _v0/core}/_data_model/importers/_rdf/_base.py +12 -12
- cognite/neat/{core → _v0/core}/_data_model/importers/_rdf/_inference2rdata_model.py +14 -14
- cognite/neat/{core → _v0/core}/_data_model/importers/_rdf/_owl2data_model.py +41 -21
- cognite/neat/{core → _v0/core}/_data_model/importers/_rdf/_shared.py +9 -9
- cognite/neat/{core → _v0/core}/_data_model/importers/_spreadsheet2data_model.py +92 -12
- cognite/neat/{core → _v0/core}/_data_model/models/__init__.py +3 -3
- cognite/neat/{core → _v0/core}/_data_model/models/_base_verified.py +5 -5
- cognite/neat/{core → _v0/core}/_data_model/models/_import_contexts.py +1 -1
- cognite/neat/{core → _v0/core}/_data_model/models/_types.py +5 -5
- cognite/neat/{core → _v0/core}/_data_model/models/conceptual/_unverified.py +16 -10
- cognite/neat/{core → _v0/core}/_data_model/models/conceptual/_validation.py +12 -12
- cognite/neat/{core → _v0/core}/_data_model/models/conceptual/_verified.py +9 -9
- cognite/neat/{core → _v0/core}/_data_model/models/data_types.py +14 -4
- cognite/neat/{core → _v0/core}/_data_model/models/entities/__init__.py +6 -0
- cognite/neat/_v0/core/_data_model/models/entities/_loaders.py +155 -0
- cognite/neat/{core → _v0/core}/_data_model/models/entities/_multi_value.py +2 -2
- cognite/neat/_v0/core/_data_model/models/entities/_restrictions.py +230 -0
- cognite/neat/{core → _v0/core}/_data_model/models/entities/_single_value.py +121 -16
- cognite/neat/{core → _v0/core}/_data_model/models/entities/_types.py +10 -0
- cognite/neat/{core → _v0/core}/_data_model/models/mapping/_classic2core.py +5 -5
- cognite/neat/{core → _v0/core}/_data_model/models/physical/__init__.py +1 -1
- cognite/neat/{core → _v0/core}/_data_model/models/physical/_exporter.py +26 -19
- cognite/neat/{core → _v0/core}/_data_model/models/physical/_unverified.py +133 -37
- cognite/neat/{core → _v0/core}/_data_model/models/physical/_validation.py +24 -20
- cognite/neat/{core → _v0/core}/_data_model/models/physical/_verified.py +95 -24
- cognite/neat/{core → _v0/core}/_data_model/transformers/_base.py +4 -4
- cognite/neat/{core → _v0/core}/_data_model/transformers/_converters.py +35 -28
- cognite/neat/{core → _v0/core}/_data_model/transformers/_mapping.py +7 -7
- cognite/neat/{core → _v0/core}/_data_model/transformers/_union_conceptual.py +5 -5
- cognite/neat/{core → _v0/core}/_data_model/transformers/_verification.py +7 -7
- cognite/neat/_v0/core/_instances/__init__.py +0 -0
- cognite/neat/{core → _v0/core}/_instances/_tracking/base.py +1 -1
- cognite/neat/{core → _v0/core}/_instances/_tracking/log.py +1 -1
- cognite/neat/{core → _v0/core}/_instances/extractors/__init__.py +3 -2
- cognite/neat/{core → _v0/core}/_instances/extractors/_base.py +6 -6
- cognite/neat/_v0/core/_instances/extractors/_classic_cdf/__init__.py +0 -0
- cognite/neat/{core → _v0/core}/_instances/extractors/_classic_cdf/_base.py +7 -7
- cognite/neat/{core → _v0/core}/_instances/extractors/_classic_cdf/_classic.py +12 -12
- cognite/neat/{core → _v0/core}/_instances/extractors/_classic_cdf/_relationships.py +3 -3
- cognite/neat/{core → _v0/core}/_instances/extractors/_classic_cdf/_sequences.py +2 -2
- cognite/neat/{core → _v0/core}/_instances/extractors/_dict.py +6 -3
- cognite/neat/{core → _v0/core}/_instances/extractors/_dms.py +6 -6
- cognite/neat/{core → _v0/core}/_instances/extractors/_dms_graph.py +11 -11
- cognite/neat/{core → _v0/core}/_instances/extractors/_mock_graph_generator.py +10 -10
- cognite/neat/{core → _v0/core}/_instances/extractors/_raw.py +3 -3
- cognite/neat/{core → _v0/core}/_instances/extractors/_rdf_file.py +7 -7
- cognite/neat/{core → _v0/core}/_instances/loaders/_base.py +5 -5
- cognite/neat/{core → _v0/core}/_instances/loaders/_rdf2dms.py +17 -17
- cognite/neat/{core → _v0/core}/_instances/loaders/_rdf_to_instance_space.py +11 -11
- cognite/neat/{core → _v0/core}/_instances/queries/_select.py +29 -3
- cognite/neat/{core → _v0/core}/_instances/queries/_update.py +1 -1
- cognite/neat/{core → _v0/core}/_instances/transformers/_base.py +4 -4
- cognite/neat/{core → _v0/core}/_instances/transformers/_classic_cdf.py +6 -6
- cognite/neat/{core → _v0/core}/_instances/transformers/_prune_graph.py +4 -4
- cognite/neat/{core → _v0/core}/_instances/transformers/_rdfpath.py +1 -1
- cognite/neat/{core → _v0/core}/_instances/transformers/_value_type.py +4 -4
- cognite/neat/{core → _v0/core}/_issues/_base.py +5 -5
- cognite/neat/{core → _v0/core}/_issues/_contextmanagers.py +1 -1
- cognite/neat/{core → _v0/core}/_issues/_factory.py +3 -3
- cognite/neat/{core → _v0/core}/_issues/errors/__init__.py +1 -1
- cognite/neat/{core → _v0/core}/_issues/errors/_external.py +1 -1
- cognite/neat/{core → _v0/core}/_issues/errors/_general.py +1 -1
- cognite/neat/{core → _v0/core}/_issues/errors/_properties.py +1 -1
- cognite/neat/{core → _v0/core}/_issues/errors/_resources.py +2 -2
- cognite/neat/{core → _v0/core}/_issues/errors/_wrapper.py +7 -3
- cognite/neat/{core → _v0/core}/_issues/warnings/__init__.py +1 -1
- cognite/neat/{core → _v0/core}/_issues/warnings/_external.py +1 -1
- cognite/neat/{core → _v0/core}/_issues/warnings/_general.py +1 -1
- cognite/neat/{core → _v0/core}/_issues/warnings/_models.py +2 -2
- cognite/neat/{core → _v0/core}/_issues/warnings/_properties.py +2 -2
- cognite/neat/{core → _v0/core}/_issues/warnings/_resources.py +1 -1
- cognite/neat/{core → _v0/core}/_issues/warnings/user_modeling.py +1 -1
- cognite/neat/{core → _v0/core}/_store/_data_model.py +12 -12
- cognite/neat/{core → _v0/core}/_store/_instance.py +43 -10
- cognite/neat/{core → _v0/core}/_store/_provenance.py +3 -3
- cognite/neat/{core → _v0/core}/_store/exceptions.py +4 -4
- cognite/neat/_v0/core/_utils/__init__.py +0 -0
- cognite/neat/{core → _v0/core}/_utils/auth.py +22 -12
- cognite/neat/{core → _v0/core}/_utils/auxiliary.py +1 -1
- cognite/neat/{core → _v0/core}/_utils/collection_.py +2 -2
- cognite/neat/{core → _v0/core}/_utils/graph_transformations_report.py +1 -1
- cognite/neat/{core → _v0/core}/_utils/rdf_.py +1 -1
- cognite/neat/{core → _v0/core}/_utils/reader/_base.py +1 -1
- cognite/neat/{core → _v0/core}/_utils/spreadsheet.py +18 -4
- cognite/neat/{core → _v0/core}/_utils/text.py +1 -1
- cognite/neat/{core → _v0/core}/_utils/upload.py +3 -3
- cognite/neat/{session → _v0}/engine/_load.py +1 -1
- cognite/neat/_v0/plugins/__init__.py +4 -0
- cognite/neat/_v0/plugins/_base.py +9 -0
- cognite/neat/_v0/plugins/_data_model.py +48 -0
- cognite/neat/{plugins → _v0/plugins}/_issues.py +1 -1
- cognite/neat/{plugins → _v0/plugins}/_manager.py +7 -16
- cognite/neat/{session → _v0/session}/_base.py +13 -15
- cognite/neat/{session → _v0/session}/_collector.py +1 -1
- cognite/neat/_v0/session/_diff.py +51 -0
- cognite/neat/{session → _v0/session}/_drop.py +3 -3
- cognite/neat/{session → _v0/session}/_explore.py +2 -2
- cognite/neat/{session → _v0/session}/_fix.py +2 -2
- cognite/neat/{session → _v0/session}/_inspect.py +3 -3
- cognite/neat/{session → _v0/session}/_mapping.py +3 -3
- cognite/neat/{session → _v0/session}/_plugin.py +4 -5
- cognite/neat/{session → _v0/session}/_prepare.py +8 -8
- cognite/neat/{session → _v0/session}/_read.py +34 -21
- cognite/neat/{session → _v0/session}/_set.py +8 -8
- cognite/neat/{session → _v0/session}/_show.py +5 -5
- cognite/neat/{session → _v0/session}/_state.py +10 -10
- cognite/neat/{session → _v0/session}/_subset.py +4 -4
- cognite/neat/{session → _v0/session}/_template.py +11 -11
- cognite/neat/{session → _v0/session}/_to.py +12 -12
- cognite/neat/{session → _v0/session}/_wizard.py +1 -1
- cognite/neat/{session → _v0/session}/exceptions.py +5 -5
- cognite/neat/_version.py +1 -1
- cognite/neat/legacy.py +6 -0
- cognite_neat-1.0.22.dist-info/METADATA +123 -0
- cognite_neat-1.0.22.dist-info/RECORD +329 -0
- cognite_neat-1.0.22.dist-info/WHEEL +4 -0
- cognite/neat/core/_data_model/models/entities/_loaders.py +0 -75
- cognite/neat/plugins/__init__.py +0 -3
- cognite/neat/plugins/data_model/importers/__init__.py +0 -5
- cognite/neat/plugins/data_model/importers/_base.py +0 -28
- cognite/neat/session/_session/_data_model/__init__.py +0 -3
- cognite/neat/session/_session/_data_model/_read.py +0 -193
- cognite/neat/session/_session/_data_model/_routes.py +0 -45
- cognite/neat/session/_session/_data_model/_show.py +0 -147
- cognite/neat/session/_session/_data_model/_write.py +0 -335
- cognite_neat-0.123.26.dist-info/METADATA +0 -144
- cognite_neat-0.123.26.dist-info/RECORD +0 -201
- cognite_neat-0.123.26.dist-info/WHEEL +0 -4
- cognite_neat-0.123.26.dist-info/licenses/LICENSE +0 -201
- /cognite/neat/{core → _client/init}/__init__.py +0 -0
- /cognite/neat/{core/_client/_api → _data_model}/__init__.py +0 -0
- /cognite/neat/{core/_client/data_classes → _data_model/deployer}/__init__.py +0 -0
- /cognite/neat/{core/_data_model → _data_model/exporters/_table_exporter}/__init__.py +0 -0
- /cognite/neat/{core/_instances → _data_model/importers/_table_importer}/__init__.py +0 -0
- /cognite/neat/{core/_instances/extractors/_classic_cdf → _data_model/models}/__init__.py +0 -0
- /cognite/neat/{core/_utils → _data_model/models/conceptual}/__init__.py +0 -0
- /cognite/neat/{plugins/data_model → _data_model/validation}/__init__.py +0 -0
- /cognite/neat/{session/_session → _session/_html}/__init__.py +0 -0
- /cognite/neat/{core → _v0/core}/_client/__init__.py +0 -0
- /cognite/neat/{core → _v0/core}/_client/data_classes/data_modeling.py +0 -0
- /cognite/neat/{core → _v0/core}/_client/data_classes/neat_sequence.py +0 -0
- /cognite/neat/{core → _v0/core}/_client/data_classes/statistics.py +0 -0
- /cognite/neat/{core → _v0/core}/_config.py +0 -0
- /cognite/neat/{core → _v0/core}/_data_model/analysis/__init__.py +0 -0
- /cognite/neat/{core → _v0/core}/_data_model/catalog/__init__.py +0 -0
- /cognite/neat/{core → _v0/core}/_data_model/catalog/classic_model.xlsx +0 -0
- /cognite/neat/{core → _v0/core}/_data_model/catalog/conceptual-imf-data-model.xlsx +0 -0
- /cognite/neat/{core → _v0/core}/_data_model/catalog/hello_world_pump.xlsx +0 -0
- /cognite/neat/{core → _v0/core}/_data_model/importers/__init__.py +0 -0
- /cognite/neat/{core → _v0/core}/_data_model/importers/_rdf/__init__.py +0 -0
- /cognite/neat/{core → _v0/core}/_data_model/models/_base_unverified.py +0 -0
- /cognite/neat/{core → _v0/core}/_data_model/models/conceptual/__init__.py +0 -0
- /cognite/neat/{core → _v0/core}/_data_model/models/entities/_constants.py +0 -0
- /cognite/neat/{core → _v0/core}/_data_model/models/entities/_wrapped.py +0 -0
- /cognite/neat/{core → _v0/core}/_data_model/models/mapping/__init__.py +0 -0
- /cognite/neat/{core → _v0/core}/_data_model/models/mapping/_classic2core.yaml +0 -0
- /cognite/neat/{core → _v0/core}/_data_model/transformers/__init__.py +0 -0
- /cognite/neat/{core → _v0/core}/_instances/_shared.py +0 -0
- /cognite/neat/{core → _v0/core}/_instances/_tracking/__init__.py +0 -0
- /cognite/neat/{core → _v0/core}/_instances/examples/Knowledge-Graph-Nordic44-dirty.xml +0 -0
- /cognite/neat/{core → _v0/core}/_instances/examples/Knowledge-Graph-Nordic44.xml +0 -0
- /cognite/neat/{core → _v0/core}/_instances/examples/__init__.py +0 -0
- /cognite/neat/{core → _v0/core}/_instances/examples/skos-capturing-sheet-wind-topics.xlsx +0 -0
- /cognite/neat/{core → _v0/core}/_instances/extractors/_classic_cdf/_assets.py +0 -0
- /cognite/neat/{core → _v0/core}/_instances/extractors/_classic_cdf/_data_sets.py +0 -0
- /cognite/neat/{core → _v0/core}/_instances/extractors/_classic_cdf/_events.py +0 -0
- /cognite/neat/{core → _v0/core}/_instances/extractors/_classic_cdf/_files.py +0 -0
- /cognite/neat/{core → _v0/core}/_instances/extractors/_classic_cdf/_labels.py +0 -0
- /cognite/neat/{core → _v0/core}/_instances/extractors/_classic_cdf/_timeseries.py +0 -0
- /cognite/neat/{core → _v0/core}/_instances/loaders/__init__.py +0 -0
- /cognite/neat/{core → _v0/core}/_instances/queries/__init__.py +0 -0
- /cognite/neat/{core → _v0/core}/_instances/queries/_base.py +0 -0
- /cognite/neat/{core → _v0/core}/_instances/queries/_queries.py +0 -0
- /cognite/neat/{core → _v0/core}/_instances/transformers/__init__.py +0 -0
- /cognite/neat/{core → _v0/core}/_issues/__init__.py +0 -0
- /cognite/neat/{core → _v0/core}/_issues/formatters.py +0 -0
- /cognite/neat/{core → _v0/core}/_shared.py +0 -0
- /cognite/neat/{core → _v0/core}/_store/__init__.py +0 -0
- /cognite/neat/{core → _v0/core}/_utils/io_.py +0 -0
- /cognite/neat/{core → _v0/core}/_utils/reader/__init__.py +0 -0
- /cognite/neat/{core → _v0/core}/_utils/tarjan.py +0 -0
- /cognite/neat/{core → _v0/core}/_utils/time_.py +0 -0
- /cognite/neat/{core → _v0/core}/_utils/xml_.py +0 -0
- /cognite/neat/{session → _v0}/engine/__init__.py +0 -0
- /cognite/neat/{session → _v0}/engine/_import.py +0 -0
- /cognite/neat/{session → _v0}/engine/_interface.py +0 -0
- /cognite/neat/{session → _v0/session}/__init__.py +0 -0
- /cognite/neat/{session → _v0/session}/_experimental.py +0 -0
- /cognite/neat/{session → _v0/session}/_state/README.md +0 -0
|
@@ -0,0 +1,431 @@
|
|
|
1
|
+
from collections.abc import Callable, Mapping, Sequence
|
|
2
|
+
from dataclasses import dataclass
|
|
3
|
+
from functools import partial
|
|
4
|
+
from typing import cast
|
|
5
|
+
|
|
6
|
+
from cognite.neat._client import NeatClient
|
|
7
|
+
from cognite.neat._data_model._constants import COGNITE_SPACES
|
|
8
|
+
from cognite.neat._data_model._shared import OnSuccessResultProducer
|
|
9
|
+
from cognite.neat._data_model._snapshot import SchemaSnapshot
|
|
10
|
+
from cognite.neat._data_model.models.dms import (
|
|
11
|
+
ContainerRequest,
|
|
12
|
+
DataModelBody,
|
|
13
|
+
RequestSchema,
|
|
14
|
+
T_DataModelResource,
|
|
15
|
+
T_ResourceId,
|
|
16
|
+
)
|
|
17
|
+
from cognite.neat._utils.collection import chunker_sequence
|
|
18
|
+
from cognite.neat._utils.http_client import (
|
|
19
|
+
FailedRequestItems,
|
|
20
|
+
FailedResponseItems,
|
|
21
|
+
ItemIDBody,
|
|
22
|
+
ItemsRequest,
|
|
23
|
+
SuccessResponseItems,
|
|
24
|
+
)
|
|
25
|
+
from cognite.neat._utils.http_client._data_classes import APIResponse
|
|
26
|
+
from cognite.neat._utils.useful_types import ModusOperandi, T_Reference
|
|
27
|
+
|
|
28
|
+
from ._differ import ItemDiffer
|
|
29
|
+
from ._differ_container import ContainerDiffer
|
|
30
|
+
from ._differ_data_model import DataModelDiffer
|
|
31
|
+
from ._differ_space import SpaceDiffer
|
|
32
|
+
from ._differ_view import ViewDiffer
|
|
33
|
+
from .data_classes import (
|
|
34
|
+
AddedField,
|
|
35
|
+
AppliedChanges,
|
|
36
|
+
ChangedFieldResult,
|
|
37
|
+
ContainerDeploymentPlan,
|
|
38
|
+
DataModelEndpoint,
|
|
39
|
+
DeploymentResult,
|
|
40
|
+
FieldChange,
|
|
41
|
+
FieldChanges,
|
|
42
|
+
HTTPChangeResult,
|
|
43
|
+
NoOpChangeResult,
|
|
44
|
+
RemovedField,
|
|
45
|
+
ResourceChange,
|
|
46
|
+
ResourceDeploymentPlan,
|
|
47
|
+
ResourceDeploymentPlanList,
|
|
48
|
+
SeverityType,
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@dataclass
|
|
53
|
+
class DeploymentOptions:
|
|
54
|
+
"""Configuration options for deployment.
|
|
55
|
+
|
|
56
|
+
Attributes:
|
|
57
|
+
dry_run (bool): If True, the deployment will be simulated without applying changes. Defaults to True.
|
|
58
|
+
auto_rollback (bool): If True, automatically roll back changes if deployment fails. Defaults to True.
|
|
59
|
+
max_severity (SeverityType): Maximum allowed severity of changes to proceed with deployment.
|
|
60
|
+
Defaults to SeverityType.SAFE.
|
|
61
|
+
modus_operandi (ModusOperandi): Deployment mode. If "additive", only add/modify resources
|
|
62
|
+
specified in the data model. If "rebuild", remove resources not present in the data model.
|
|
63
|
+
Defaults to "additive".
|
|
64
|
+
"""
|
|
65
|
+
|
|
66
|
+
dry_run: bool = True
|
|
67
|
+
auto_rollback: bool = True
|
|
68
|
+
drop_data: bool = False
|
|
69
|
+
max_severity: SeverityType = SeverityType.WARNING
|
|
70
|
+
modus_operandi: ModusOperandi = "additive"
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
class SchemaDeployer(OnSuccessResultProducer):
|
|
74
|
+
INDEX_DELETE_BATCH_SIZE = 10
|
|
75
|
+
CONSTRAINT_DELETE_BATCH_SIZE = 10
|
|
76
|
+
|
|
77
|
+
def __init__(self, client: NeatClient, options: DeploymentOptions | None = None) -> None:
|
|
78
|
+
super().__init__()
|
|
79
|
+
self.client: NeatClient = client
|
|
80
|
+
self.options: DeploymentOptions = options or DeploymentOptions()
|
|
81
|
+
|
|
82
|
+
def run(self, data_model: RequestSchema) -> None:
|
|
83
|
+
if self._results is not None:
|
|
84
|
+
raise RuntimeError("SchemaDeployer has already been run.")
|
|
85
|
+
self._results = self.deploy(data_model)
|
|
86
|
+
|
|
87
|
+
def deploy(self, data_model: RequestSchema) -> DeploymentResult:
|
|
88
|
+
# Step 1: Fetch current CDF state
|
|
89
|
+
snapshot = SchemaSnapshot.fetch_cdf_data_model(self.client, data_model)
|
|
90
|
+
|
|
91
|
+
# Step 2: Create deployment plan by comparing local vs cdf
|
|
92
|
+
plan = self.create_deployment_plan(snapshot, data_model)
|
|
93
|
+
|
|
94
|
+
# Step 3: Adjust plan based on modus operandi
|
|
95
|
+
if self.options.modus_operandi == "additive":
|
|
96
|
+
# Filter out deletions and removals in additive mode
|
|
97
|
+
plan = plan.consolidate_changes()
|
|
98
|
+
elif self.options.modus_operandi == "rebuild":
|
|
99
|
+
# Breaking changes are forced by deleting and recreating resources
|
|
100
|
+
# Containers are treated as additive unless drop_data is specified
|
|
101
|
+
plan = plan.force_changes(self.options.drop_data)
|
|
102
|
+
else:
|
|
103
|
+
raise NotImplementedError(f"Unsupported modus operandi: {self.options.modus_operandi!r}")
|
|
104
|
+
|
|
105
|
+
if not self.should_proceed_to_deploy(plan):
|
|
106
|
+
# Step 4: Check if deployment should proceed
|
|
107
|
+
return DeploymentResult(status="failure", plan=list(plan), snapshot=snapshot)
|
|
108
|
+
elif self.options.dry_run:
|
|
109
|
+
# Step 5: If dry-run, return plan without applying
|
|
110
|
+
return DeploymentResult(status="pending", plan=list(plan), snapshot=snapshot)
|
|
111
|
+
|
|
112
|
+
# Step 6: Apply changes
|
|
113
|
+
changes = self.apply_changes(plan)
|
|
114
|
+
|
|
115
|
+
# Step 7: Rollback if failed and auto_rollback is enabled
|
|
116
|
+
if not changes.is_success and self.options.auto_rollback:
|
|
117
|
+
recovery = self.apply_changes(changes.as_recovery_plan())
|
|
118
|
+
return DeploymentResult(
|
|
119
|
+
status="recovered" if recovery.is_success else "recovery_failed",
|
|
120
|
+
plan=list(plan),
|
|
121
|
+
snapshot=snapshot,
|
|
122
|
+
responses=changes,
|
|
123
|
+
recovery=recovery,
|
|
124
|
+
)
|
|
125
|
+
return DeploymentResult(
|
|
126
|
+
status="success" if changes.is_success else "partial", plan=list(plan), snapshot=snapshot, responses=changes
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
def create_deployment_plan(self, snapshot: SchemaSnapshot, data_model: RequestSchema) -> ResourceDeploymentPlanList:
|
|
130
|
+
return ResourceDeploymentPlanList(
|
|
131
|
+
[
|
|
132
|
+
self._create_resource_plan(
|
|
133
|
+
snapshot.spaces,
|
|
134
|
+
data_model.spaces,
|
|
135
|
+
"spaces",
|
|
136
|
+
SpaceDiffer(),
|
|
137
|
+
skip_criteria=partial(self._skip_resource, model_space=data_model.data_model.space),
|
|
138
|
+
),
|
|
139
|
+
self._create_resource_plan(
|
|
140
|
+
snapshot.containers,
|
|
141
|
+
data_model.containers,
|
|
142
|
+
"containers",
|
|
143
|
+
ContainerDiffer(),
|
|
144
|
+
ContainerDeploymentPlan,
|
|
145
|
+
skip_criteria=partial(self._skip_resource, model_space=data_model.data_model.space),
|
|
146
|
+
),
|
|
147
|
+
self._create_resource_plan(
|
|
148
|
+
snapshot.views,
|
|
149
|
+
data_model.views,
|
|
150
|
+
"views",
|
|
151
|
+
ViewDiffer(
|
|
152
|
+
current_container_map=snapshot.containers,
|
|
153
|
+
new_container_map={container.as_reference(): container for container in data_model.containers},
|
|
154
|
+
),
|
|
155
|
+
skip_criteria=partial(self._skip_resource, model_space=data_model.data_model.space),
|
|
156
|
+
),
|
|
157
|
+
self._create_resource_plan(
|
|
158
|
+
snapshot.data_model,
|
|
159
|
+
[data_model.data_model],
|
|
160
|
+
"datamodels",
|
|
161
|
+
DataModelDiffer(),
|
|
162
|
+
skip_criteria=partial(self._skip_resource, model_space=data_model.data_model.space),
|
|
163
|
+
),
|
|
164
|
+
]
|
|
165
|
+
)
|
|
166
|
+
|
|
167
|
+
def _create_resource_plan(
|
|
168
|
+
self,
|
|
169
|
+
current_resources: dict[T_ResourceId, T_DataModelResource],
|
|
170
|
+
new_resources: list[T_DataModelResource],
|
|
171
|
+
endpoint: DataModelEndpoint,
|
|
172
|
+
differ: ItemDiffer[T_DataModelResource],
|
|
173
|
+
plan_type: type[ResourceDeploymentPlan[T_ResourceId, T_DataModelResource]] = ResourceDeploymentPlan,
|
|
174
|
+
skip_criteria: Callable[[T_ResourceId], str | None] | None = None,
|
|
175
|
+
) -> ResourceDeploymentPlan[T_ResourceId, T_DataModelResource]:
|
|
176
|
+
resources: list[ResourceChange[T_ResourceId, T_DataModelResource]] = []
|
|
177
|
+
for new_resource in new_resources:
|
|
178
|
+
# We know that .as_reference() will return T_ResourceId
|
|
179
|
+
ref = cast(T_ResourceId, new_resource.as_reference())
|
|
180
|
+
if skip_criteria is not None and (reason := skip_criteria(ref)):
|
|
181
|
+
resources.append(ResourceChange(resource_id=ref, new_value=None, current_value=None, message=reason))
|
|
182
|
+
continue
|
|
183
|
+
if ref not in current_resources:
|
|
184
|
+
resources.append(ResourceChange(resource_id=ref, new_value=new_resource))
|
|
185
|
+
continue
|
|
186
|
+
current_resource = current_resources[ref]
|
|
187
|
+
diffs = differ.diff(current_resource, new_resource)
|
|
188
|
+
if (
|
|
189
|
+
isinstance(current_resource, ContainerRequest)
|
|
190
|
+
and isinstance(new_resource, ContainerRequest)
|
|
191
|
+
and self.options.modus_operandi == "additive"
|
|
192
|
+
):
|
|
193
|
+
# In additive mode, changes to constraints and indexes require removal and re-adding
|
|
194
|
+
# In rebuild mode, all changes are forced via deletion and re-adding
|
|
195
|
+
diffs = self.remove_readd_modified_indexes_and_constraints(diffs, current_resource, new_resource)
|
|
196
|
+
resources.append(
|
|
197
|
+
ResourceChange(resource_id=ref, new_value=new_resource, current_value=current_resource, changes=diffs)
|
|
198
|
+
)
|
|
199
|
+
|
|
200
|
+
return plan_type(endpoint=endpoint, resources=resources)
|
|
201
|
+
|
|
202
|
+
@classmethod
|
|
203
|
+
def remove_readd_modified_indexes_and_constraints(
|
|
204
|
+
cls, diffs: list[FieldChange], current_resource: ContainerRequest, new_resource: ContainerRequest
|
|
205
|
+
) -> list[FieldChange]:
|
|
206
|
+
"""Constraints and indexes cannot be modified directly; they must be removed and re-added.
|
|
207
|
+
|
|
208
|
+
Args:
|
|
209
|
+
diffs: The list of field changes detected by the differ.
|
|
210
|
+
current_resource: The current state of the container.
|
|
211
|
+
new_resource: The desired state of the container.
|
|
212
|
+
Returns:
|
|
213
|
+
A modified list of field changes with constraints and indexes handled appropriately.
|
|
214
|
+
"""
|
|
215
|
+
modified_diffs: list[FieldChange] = []
|
|
216
|
+
for diff in diffs:
|
|
217
|
+
if (diff.field_path.startswith("constraints") or diff.field_path.startswith("indexes")) and isinstance(
|
|
218
|
+
diff, FieldChanges
|
|
219
|
+
):
|
|
220
|
+
if "." not in diff.field_path:
|
|
221
|
+
# Should not happen, but just in case
|
|
222
|
+
raise RuntimeError("Bug in Neat. Malformed field path for constraint/index change.")
|
|
223
|
+
# Field type is either "constraints" or "indexes"
|
|
224
|
+
field_type, identifier, *_ = diff.field_path.split(".", maxsplit=2)
|
|
225
|
+
# Mark for removal
|
|
226
|
+
modified_diffs.append(
|
|
227
|
+
RemovedField(
|
|
228
|
+
field_path=f"{field_type}.{identifier}",
|
|
229
|
+
item_severity=SeverityType.WARNING,
|
|
230
|
+
current_value=getattr(current_resource, field_type)[identifier],
|
|
231
|
+
)
|
|
232
|
+
)
|
|
233
|
+
# Mark for addition
|
|
234
|
+
modified_diffs.append(
|
|
235
|
+
AddedField(
|
|
236
|
+
field_path=f"{field_type}.{identifier}",
|
|
237
|
+
item_severity=SeverityType.SAFE,
|
|
238
|
+
new_value=getattr(new_resource, field_type)[identifier],
|
|
239
|
+
)
|
|
240
|
+
)
|
|
241
|
+
else:
|
|
242
|
+
modified_diffs.append(diff)
|
|
243
|
+
return modified_diffs
|
|
244
|
+
|
|
245
|
+
@classmethod
|
|
246
|
+
def _skip_resource(cls, resource_id: T_ResourceId, model_space: str) -> str | None:
|
|
247
|
+
"""Checks if a resource should be skipped based on its space.
|
|
248
|
+
|
|
249
|
+
Args:
|
|
250
|
+
resource_id: The ID of the resource to check.
|
|
251
|
+
model_space: The space of the data model.
|
|
252
|
+
|
|
253
|
+
Returns:
|
|
254
|
+
A reason for skipping if the resource space does not match the model space, otherwise None.
|
|
255
|
+
"""
|
|
256
|
+
if resource_id.space in COGNITE_SPACES:
|
|
257
|
+
return f"Skipping resource as it is in the reserved Cognite space '{resource_id.space}'."
|
|
258
|
+
elif resource_id.space != model_space:
|
|
259
|
+
return (
|
|
260
|
+
f"Skipping resource as it is in the space '{resource_id.space}'"
|
|
261
|
+
f" not matching data model space '{model_space}'."
|
|
262
|
+
)
|
|
263
|
+
return None
|
|
264
|
+
|
|
265
|
+
def should_proceed_to_deploy(self, plan: Sequence[ResourceDeploymentPlan]) -> bool:
|
|
266
|
+
max_severity_in_plan = SeverityType.max_severity(
|
|
267
|
+
[change.severity for resource_plan in plan for change in resource_plan.resources],
|
|
268
|
+
default=SeverityType.SAFE,
|
|
269
|
+
)
|
|
270
|
+
return max_severity_in_plan.value <= self.options.max_severity.value
|
|
271
|
+
|
|
272
|
+
def apply_changes(self, plan: Sequence[ResourceDeploymentPlan]) -> AppliedChanges:
|
|
273
|
+
"""Applies the given deployment plan to CDF by making the necessary API calls.
|
|
274
|
+
|
|
275
|
+
Args:
|
|
276
|
+
plan (list[ResourceDeploymentPlan]): The deployment plan to apply.
|
|
277
|
+
|
|
278
|
+
Returns:
|
|
279
|
+
AppliedChanges: The result of applying the changes.
|
|
280
|
+
"""
|
|
281
|
+
applied_changes = AppliedChanges()
|
|
282
|
+
# If any HTTP request fails, the skip_message will be set and subsequent operations will be skipped
|
|
283
|
+
failure_message: str | None = None
|
|
284
|
+
for resource in reversed(plan):
|
|
285
|
+
if failure_message is None:
|
|
286
|
+
deletions = self._delete_items(resource)
|
|
287
|
+
applied_changes.deletions.extend(deletions)
|
|
288
|
+
if any(not deletion.is_success for deletion in deletions):
|
|
289
|
+
failure_message = f"Skipping due to {resource.endpoint} deletions failing."
|
|
290
|
+
else:
|
|
291
|
+
applied_changes.skipped.extend(
|
|
292
|
+
[
|
|
293
|
+
NoOpChangeResult(endpoint=resource.endpoint, change=change, reason=failure_message)
|
|
294
|
+
for change in resource.to_delete
|
|
295
|
+
]
|
|
296
|
+
)
|
|
297
|
+
|
|
298
|
+
for resource in plan:
|
|
299
|
+
if failure_message is None:
|
|
300
|
+
if isinstance(resource, ContainerDeploymentPlan):
|
|
301
|
+
# Note that we continue to deploy even if removing constraints/indexes fail,
|
|
302
|
+
# as the creation/update of views and data models will still succeed.
|
|
303
|
+
applied_changes.changed_fields.extend(self._remove_container_constraints(resource))
|
|
304
|
+
applied_changes.changed_fields.extend(self._remove_container_indexes(resource))
|
|
305
|
+
|
|
306
|
+
creations, updated = self._upsert_items(resource)
|
|
307
|
+
applied_changes.created.extend(creations)
|
|
308
|
+
applied_changes.updated.extend(updated)
|
|
309
|
+
if any(not change.is_success for change in creations + updated):
|
|
310
|
+
failure_message = f"Skipping due to {resource.endpoint} upsert failing."
|
|
311
|
+
else:
|
|
312
|
+
applied_changes.skipped.extend(
|
|
313
|
+
[
|
|
314
|
+
NoOpChangeResult(endpoint=resource.endpoint, change=change, reason=failure_message)
|
|
315
|
+
for change in resource.to_upsert
|
|
316
|
+
]
|
|
317
|
+
)
|
|
318
|
+
|
|
319
|
+
applied_changes.unchanged.extend(
|
|
320
|
+
[
|
|
321
|
+
NoOpChangeResult(endpoint=resource.endpoint, change=change, reason="No changes detected.")
|
|
322
|
+
for change in resource.unchanged
|
|
323
|
+
]
|
|
324
|
+
)
|
|
325
|
+
applied_changes.skipped.extend(
|
|
326
|
+
[
|
|
327
|
+
NoOpChangeResult(endpoint=resource.endpoint, change=change, reason=change.message or "Unknown")
|
|
328
|
+
for change in resource.skip
|
|
329
|
+
]
|
|
330
|
+
)
|
|
331
|
+
return applied_changes
|
|
332
|
+
|
|
333
|
+
def _delete_items(self, resource: ResourceDeploymentPlan) -> list[HTTPChangeResult]:
|
|
334
|
+
to_delete_by_id = {change.resource_id: change for change in resource.to_delete}
|
|
335
|
+
if not to_delete_by_id:
|
|
336
|
+
return []
|
|
337
|
+
responses = self.client.http_client.request_with_retries(
|
|
338
|
+
ItemsRequest(
|
|
339
|
+
endpoint_url=self.client.config.create_api_url(f"/models/{resource.endpoint}/delete"),
|
|
340
|
+
method="POST",
|
|
341
|
+
body=ItemIDBody(items=list(to_delete_by_id.keys())),
|
|
342
|
+
)
|
|
343
|
+
)
|
|
344
|
+
return self._process_resource_responses(responses, to_delete_by_id, resource.endpoint)
|
|
345
|
+
|
|
346
|
+
def _upsert_items(self, resource: ResourceDeploymentPlan) -> tuple[list[HTTPChangeResult], list[HTTPChangeResult]]:
|
|
347
|
+
to_upsert = [
|
|
348
|
+
resource_change.new_value for resource_change in resource.to_upsert if resource_change.new_value is not None
|
|
349
|
+
]
|
|
350
|
+
if not to_upsert:
|
|
351
|
+
return [], []
|
|
352
|
+
responses = self.client.http_client.request_with_retries(
|
|
353
|
+
ItemsRequest(
|
|
354
|
+
endpoint_url=self.client.config.create_api_url(f"/models/{resource.endpoint}"),
|
|
355
|
+
method="POST",
|
|
356
|
+
body=DataModelBody(items=to_upsert),
|
|
357
|
+
)
|
|
358
|
+
)
|
|
359
|
+
to_create_by_id = {rc.resource_id: rc for rc in resource.to_create}
|
|
360
|
+
create_result = self._process_resource_responses(responses, to_create_by_id, resource.endpoint)
|
|
361
|
+
to_update_by_id = {rc.resource_id: rc for rc in resource.to_update}
|
|
362
|
+
update_result = self._process_resource_responses(responses, to_update_by_id, resource.endpoint)
|
|
363
|
+
return create_result, update_result
|
|
364
|
+
|
|
365
|
+
def _remove_container_indexes(self, resource: ContainerDeploymentPlan) -> list[ChangedFieldResult]:
|
|
366
|
+
return self._remove_container_fields(
|
|
367
|
+
resource.indexes_to_remove,
|
|
368
|
+
"/models/containers/indexes/delete",
|
|
369
|
+
self.INDEX_DELETE_BATCH_SIZE,
|
|
370
|
+
)
|
|
371
|
+
|
|
372
|
+
def _remove_container_constraints(self, resource: ContainerDeploymentPlan) -> list[ChangedFieldResult]:
|
|
373
|
+
return self._remove_container_fields(
|
|
374
|
+
resource.constraints_to_remove,
|
|
375
|
+
"/models/containers/constraints/delete",
|
|
376
|
+
self.CONSTRAINT_DELETE_BATCH_SIZE,
|
|
377
|
+
)
|
|
378
|
+
|
|
379
|
+
def _remove_container_fields(
|
|
380
|
+
self,
|
|
381
|
+
fields_to_remove: Mapping[T_Reference, FieldChange],
|
|
382
|
+
endpoint: str,
|
|
383
|
+
batch_size: int,
|
|
384
|
+
) -> list[ChangedFieldResult]:
|
|
385
|
+
if not fields_to_remove:
|
|
386
|
+
return []
|
|
387
|
+
results: list[ChangedFieldResult] = []
|
|
388
|
+
for batch in chunker_sequence(list(fields_to_remove.keys()), batch_size):
|
|
389
|
+
responses = self.client.http_client.request_with_retries(
|
|
390
|
+
ItemsRequest(
|
|
391
|
+
endpoint_url=self.client.config.create_api_url(endpoint),
|
|
392
|
+
method="POST",
|
|
393
|
+
body=ItemIDBody(items=batch),
|
|
394
|
+
)
|
|
395
|
+
)
|
|
396
|
+
results.extend(self._process_field_responses(responses, fields_to_remove))
|
|
397
|
+
return results
|
|
398
|
+
|
|
399
|
+
@classmethod
|
|
400
|
+
def _process_resource_responses(
|
|
401
|
+
cls, responses: APIResponse, change_by_id: dict[T_ResourceId, ResourceChange], endpoint: DataModelEndpoint
|
|
402
|
+
) -> list[HTTPChangeResult]:
|
|
403
|
+
results: list[HTTPChangeResult] = []
|
|
404
|
+
for response in responses:
|
|
405
|
+
if isinstance(response, SuccessResponseItems | FailedResponseItems | FailedRequestItems):
|
|
406
|
+
for id in response.ids:
|
|
407
|
+
if id not in change_by_id:
|
|
408
|
+
continue
|
|
409
|
+
results.append(HTTPChangeResult(change=change_by_id[id], http_message=response, endpoint=endpoint))
|
|
410
|
+
else:
|
|
411
|
+
# This should never happen as we do a ItemsRequest should always return ItemMessage responses
|
|
412
|
+
raise ValueError("Bug in Neat. Got an unexpected response type.")
|
|
413
|
+
return results
|
|
414
|
+
|
|
415
|
+
@classmethod
|
|
416
|
+
def _process_field_responses(
|
|
417
|
+
cls, responses: APIResponse, change_by_id: Mapping[T_Reference, FieldChange]
|
|
418
|
+
) -> list[ChangedFieldResult]:
|
|
419
|
+
results: list[ChangedFieldResult] = []
|
|
420
|
+
for response in responses:
|
|
421
|
+
if isinstance(response, SuccessResponseItems | FailedResponseItems | FailedRequestItems):
|
|
422
|
+
for id in response.ids:
|
|
423
|
+
if id not in change_by_id:
|
|
424
|
+
continue
|
|
425
|
+
results.append(
|
|
426
|
+
ChangedFieldResult(resource_id=id, field_change=change_by_id[id], http_message=response)
|
|
427
|
+
)
|
|
428
|
+
else:
|
|
429
|
+
# This should never happen as we do a ItemsRequest should always return ItemMessage responses
|
|
430
|
+
raise RuntimeError("Bug in Neat. Got an unexpected response type.")
|
|
431
|
+
return results
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
from ._api_exporter import DMSAPIExporter, DMSAPIJSONExporter, DMSAPIYAMLExporter
|
|
2
|
+
from ._base import DMSExporter, DMSFileExporter
|
|
3
|
+
from ._table_exporter.exporter import DMSExcelExporter, DMSTableExporter, DMSTableJSONExporter, DMSTableYamlExporter
|
|
4
|
+
|
|
5
|
+
__all__ = [
|
|
6
|
+
"DMSAPIExporter",
|
|
7
|
+
"DMSAPIJSONExporter",
|
|
8
|
+
"DMSAPIYAMLExporter",
|
|
9
|
+
"DMSExcelExporter",
|
|
10
|
+
"DMSExporter",
|
|
11
|
+
"DMSFileExporter",
|
|
12
|
+
"DMSTableExporter",
|
|
13
|
+
"DMSTableJSONExporter",
|
|
14
|
+
"DMSTableYamlExporter",
|
|
15
|
+
]
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
|
|
3
|
+
import yaml
|
|
4
|
+
|
|
5
|
+
from cognite.neat._data_model.exporters._base import DMSExporter, DMSFileExporter
|
|
6
|
+
from cognite.neat._data_model.models.dms import RequestSchema
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class DMSAPIExporter(DMSExporter[RequestSchema]):
|
|
10
|
+
def export(self, data_model: RequestSchema) -> RequestSchema:
|
|
11
|
+
return data_model
|
|
12
|
+
|
|
13
|
+
def export_to_file(self, data_model: RequestSchema, file_path: Path) -> None:
|
|
14
|
+
raise RuntimeError(f"{type(self).__name__} does not support export_to_file method.")
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class DMSAPIYAMLExporter(DMSAPIExporter, DMSFileExporter[RequestSchema]):
|
|
18
|
+
def export_to_file(self, data_model: RequestSchema, file_path: Path) -> None:
|
|
19
|
+
"""Export the data model to a YAML file in API format."""
|
|
20
|
+
if file_path.suffix.lower() not in {".yaml", ".yml"}:
|
|
21
|
+
raise ValueError("The file path must have a .yaml or .yml extension.")
|
|
22
|
+
|
|
23
|
+
api_format = data_model.model_dump(mode="json", by_alias=True)
|
|
24
|
+
file_path.write_text(yaml.safe_dump(api_format, sort_keys=False), encoding=self.ENCODING, newline=self.NEW_LINE)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class DMSAPIJSONExporter(DMSAPIExporter, DMSFileExporter[RequestSchema]):
|
|
28
|
+
def export_to_file(self, data_model: RequestSchema, file_path: Path) -> None:
|
|
29
|
+
"""Export the data model to a JSON file in API format."""
|
|
30
|
+
if file_path.suffix.lower() != ".json":
|
|
31
|
+
raise ValueError("The file path must have a .json extension.")
|
|
32
|
+
|
|
33
|
+
file_path.write_text(
|
|
34
|
+
data_model.model_dump_json(by_alias=True),
|
|
35
|
+
encoding=self.ENCODING,
|
|
36
|
+
newline=self.NEW_LINE,
|
|
37
|
+
)
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
from abc import ABC, abstractmethod
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
from typing import Generic, TypeVar
|
|
4
|
+
|
|
5
|
+
from cognite.neat._data_model.models.dms import RequestSchema
|
|
6
|
+
|
|
7
|
+
T_Export = TypeVar("T_Export")
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class DMSExporter(ABC, Generic[T_Export]):
|
|
11
|
+
"""This is the base class for all DMS exporters."""
|
|
12
|
+
|
|
13
|
+
NEW_LINE = "\n"
|
|
14
|
+
ENCODING = "utf-8"
|
|
15
|
+
|
|
16
|
+
@abstractmethod
|
|
17
|
+
def export(self, data_model: RequestSchema) -> T_Export:
|
|
18
|
+
raise NotImplementedError()
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class DMSFileExporter(DMSExporter[T_Export], ABC):
|
|
22
|
+
@abstractmethod
|
|
23
|
+
def export_to_file(self, data_model: RequestSchema, file_path: Path) -> None:
|
|
24
|
+
raise NotImplementedError()
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import json
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
from typing import cast
|
|
4
|
+
|
|
5
|
+
import yaml
|
|
6
|
+
|
|
7
|
+
from cognite.neat._data_model.exporters._base import DMSExporter, DMSFileExporter
|
|
8
|
+
from cognite.neat._data_model.importers._table_importer.data_classes import DMSProperty, TableDMS
|
|
9
|
+
from cognite.neat._data_model.models.dms import RequestSchema
|
|
10
|
+
from cognite.neat._utils.useful_types import DataModelTableType
|
|
11
|
+
|
|
12
|
+
from .workbook import WorkbookCreator, WorkbookOptions
|
|
13
|
+
from .writer import DMSTableWriter
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class DMSTableExporter(DMSExporter[DataModelTableType]):
|
|
17
|
+
"""Exports DMS to a table structure.
|
|
18
|
+
|
|
19
|
+
The tables can are expected to be a dictionary where the keys are the table names and the values
|
|
20
|
+
are lists of dictionaries representing the rows in the table.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
class Sheets:
|
|
24
|
+
properties = cast(str, TableDMS.model_fields["properties"].validation_alias)
|
|
25
|
+
|
|
26
|
+
def __init__(self, exclude_none: bool = False, skip_properties_in_other_spaces: bool = True) -> None:
|
|
27
|
+
self._exclude_none = exclude_none
|
|
28
|
+
self._skip_properties_in_other_spaces = skip_properties_in_other_spaces
|
|
29
|
+
|
|
30
|
+
def export(self, data_model: RequestSchema) -> DataModelTableType:
|
|
31
|
+
model = data_model.data_model
|
|
32
|
+
tables = DMSTableWriter(model.space, model.version, self._skip_properties_in_other_spaces).write_tables(
|
|
33
|
+
data_model
|
|
34
|
+
)
|
|
35
|
+
exclude: set[str] = set()
|
|
36
|
+
if self._exclude_none:
|
|
37
|
+
if not tables.enum:
|
|
38
|
+
exclude.add("enum")
|
|
39
|
+
if not tables.nodes:
|
|
40
|
+
exclude.add("nodes")
|
|
41
|
+
if not tables.containers:
|
|
42
|
+
exclude.add("containers")
|
|
43
|
+
|
|
44
|
+
output = tables.model_dump(mode="json", by_alias=True, exclude_none=self._exclude_none, exclude=exclude)
|
|
45
|
+
# When we have exclude_none we only want to exclude none of optional properties, not required.
|
|
46
|
+
# Thus, we do the implementation below
|
|
47
|
+
required_properties = [
|
|
48
|
+
field_.serialization_alias for field_ in DMSProperty.model_fields.values() if field_.is_required()
|
|
49
|
+
]
|
|
50
|
+
for row in output[self.Sheets.properties]:
|
|
51
|
+
for prop in required_properties:
|
|
52
|
+
if prop not in row:
|
|
53
|
+
row[prop] = None
|
|
54
|
+
return output
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class DMSTableYamlExporter(DMSTableExporter, DMSFileExporter[DataModelTableType]):
|
|
58
|
+
"""Exports DMS to YAML."""
|
|
59
|
+
|
|
60
|
+
def __init__(self) -> None:
|
|
61
|
+
super().__init__(exclude_none=True)
|
|
62
|
+
|
|
63
|
+
def export_to_file(self, data_model: RequestSchema, file_path: Path) -> None:
|
|
64
|
+
"""Exports the data model as a flat YAML file, which is identical to the spreadsheet representation
|
|
65
|
+
|
|
66
|
+
Args:
|
|
67
|
+
data_model (RequestSchema): The data model to export.
|
|
68
|
+
file_path (Path): The path to the YAML file to create.
|
|
69
|
+
"""
|
|
70
|
+
table_format = self.export(data_model)
|
|
71
|
+
file_path.write_text(
|
|
72
|
+
yaml.safe_dump(table_format, sort_keys=False), encoding=self.ENCODING, newline=self.NEW_LINE
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
class DMSTableJSONExporter(DMSTableExporter, DMSFileExporter[DataModelTableType]):
|
|
77
|
+
"""Exports DMS to JSON."""
|
|
78
|
+
|
|
79
|
+
def __init__(self) -> None:
|
|
80
|
+
super().__init__(exclude_none=True)
|
|
81
|
+
|
|
82
|
+
def export_to_file(self, data_model: RequestSchema, file_path: Path) -> None:
|
|
83
|
+
"""Exports the data model as a flat JSON file, which is identical to the spreadsheet representation
|
|
84
|
+
|
|
85
|
+
Args:
|
|
86
|
+
data_model (RequestSchema): The data model to export.
|
|
87
|
+
file_path (Path): The path to the JSON file to create.
|
|
88
|
+
"""
|
|
89
|
+
table_format = self.export(data_model)
|
|
90
|
+
file_path.write_text(json.dumps(table_format), encoding=self.ENCODING, newline=self.NEW_LINE)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
class DMSExcelExporter(DMSTableExporter, DMSFileExporter[DataModelTableType]):
|
|
94
|
+
"""Exports DMS to Excel file."""
|
|
95
|
+
|
|
96
|
+
def __init__(self, options: WorkbookOptions | None = None) -> None:
|
|
97
|
+
self._options = options or WorkbookOptions()
|
|
98
|
+
super().__init__(
|
|
99
|
+
exclude_none=False, skip_properties_in_other_spaces=self._options.skip_properties_in_other_spaces
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
def export_to_file(self, data_model: RequestSchema, file_path: Path) -> None:
|
|
103
|
+
"""Exports the data model as a Excel file.
|
|
104
|
+
|
|
105
|
+
Args:
|
|
106
|
+
data_model (RequestSchema): The data model to export.
|
|
107
|
+
file_path (Path): The path to the Excel file to create.
|
|
108
|
+
options (WorkbookOptions | None): Options for creating the workbook.
|
|
109
|
+
"""
|
|
110
|
+
table_format = self.export(data_model)
|
|
111
|
+
workbook = WorkbookCreator(self._options).create_workbook(table_format)
|
|
112
|
+
try:
|
|
113
|
+
workbook.save(file_path)
|
|
114
|
+
finally:
|
|
115
|
+
workbook.close()
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
class DMSCsvExporter(DMSTableExporter, DMSFileExporter[DataModelTableType]):
|
|
119
|
+
"""Exports DMS to CSV files in a directory."""
|
|
120
|
+
|
|
121
|
+
def export_to_file(self, data_model: RequestSchema, directory_path: Path) -> None:
|
|
122
|
+
"""Exports the data model as a set of CSV files, one for each table.
|
|
123
|
+
|
|
124
|
+
Args:
|
|
125
|
+
data_model (RequestSchema): The data model to export.
|
|
126
|
+
directory_path (Path): The path to the directory to create the CSV files in.
|
|
127
|
+
"""
|
|
128
|
+
raise NotImplementedError()
|