cognite-neat 0.123.26__py3-none-any.whl → 1.0.22__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (341) hide show
  1. cognite/neat/__init__.py +4 -3
  2. cognite/neat/_client/__init__.py +5 -0
  3. cognite/neat/_client/api.py +8 -0
  4. cognite/neat/_client/client.py +21 -0
  5. cognite/neat/_client/config.py +40 -0
  6. cognite/neat/_client/containers_api.py +138 -0
  7. cognite/neat/_client/data_classes.py +44 -0
  8. cognite/neat/_client/data_model_api.py +115 -0
  9. cognite/neat/_client/init/credentials.py +70 -0
  10. cognite/neat/_client/init/env_vars.py +131 -0
  11. cognite/neat/_client/init/main.py +51 -0
  12. cognite/neat/_client/spaces_api.py +115 -0
  13. cognite/neat/_client/statistics_api.py +24 -0
  14. cognite/neat/_client/views_api.py +144 -0
  15. cognite/neat/_config.py +266 -0
  16. cognite/neat/_data_model/_analysis.py +571 -0
  17. cognite/neat/_data_model/_constants.py +74 -0
  18. cognite/neat/_data_model/_identifiers.py +61 -0
  19. cognite/neat/_data_model/_shared.py +41 -0
  20. cognite/neat/_data_model/_snapshot.py +134 -0
  21. cognite/neat/_data_model/deployer/_differ.py +140 -0
  22. cognite/neat/_data_model/deployer/_differ_container.py +360 -0
  23. cognite/neat/_data_model/deployer/_differ_data_model.py +54 -0
  24. cognite/neat/_data_model/deployer/_differ_space.py +9 -0
  25. cognite/neat/_data_model/deployer/_differ_view.py +299 -0
  26. cognite/neat/_data_model/deployer/data_classes.py +644 -0
  27. cognite/neat/_data_model/deployer/deployer.py +431 -0
  28. cognite/neat/_data_model/exporters/__init__.py +15 -0
  29. cognite/neat/_data_model/exporters/_api_exporter.py +37 -0
  30. cognite/neat/_data_model/exporters/_base.py +24 -0
  31. cognite/neat/_data_model/exporters/_table_exporter/exporter.py +128 -0
  32. cognite/neat/_data_model/exporters/_table_exporter/workbook.py +409 -0
  33. cognite/neat/_data_model/exporters/_table_exporter/writer.py +480 -0
  34. cognite/neat/_data_model/importers/__init__.py +5 -0
  35. cognite/neat/_data_model/importers/_api_importer.py +166 -0
  36. cognite/neat/_data_model/importers/_base.py +16 -0
  37. cognite/neat/_data_model/importers/_table_importer/data_classes.py +344 -0
  38. cognite/neat/_data_model/importers/_table_importer/importer.py +192 -0
  39. cognite/neat/_data_model/importers/_table_importer/reader.py +1102 -0
  40. cognite/neat/_data_model/importers/_table_importer/source.py +94 -0
  41. cognite/neat/_data_model/models/conceptual/_base.py +18 -0
  42. cognite/neat/_data_model/models/conceptual/_concept.py +67 -0
  43. cognite/neat/_data_model/models/conceptual/_data_model.py +51 -0
  44. cognite/neat/_data_model/models/conceptual/_properties.py +104 -0
  45. cognite/neat/_data_model/models/conceptual/_property.py +105 -0
  46. cognite/neat/_data_model/models/dms/__init__.py +206 -0
  47. cognite/neat/_data_model/models/dms/_base.py +31 -0
  48. cognite/neat/_data_model/models/dms/_constants.py +48 -0
  49. cognite/neat/_data_model/models/dms/_constraints.py +42 -0
  50. cognite/neat/_data_model/models/dms/_container.py +159 -0
  51. cognite/neat/_data_model/models/dms/_data_model.py +95 -0
  52. cognite/neat/_data_model/models/dms/_data_types.py +195 -0
  53. cognite/neat/_data_model/models/dms/_http.py +28 -0
  54. cognite/neat/_data_model/models/dms/_indexes.py +30 -0
  55. cognite/neat/_data_model/models/dms/_limits.py +96 -0
  56. cognite/neat/_data_model/models/dms/_references.py +141 -0
  57. cognite/neat/_data_model/models/dms/_schema.py +18 -0
  58. cognite/neat/_data_model/models/dms/_space.py +48 -0
  59. cognite/neat/_data_model/models/dms/_types.py +17 -0
  60. cognite/neat/_data_model/models/dms/_view_filter.py +310 -0
  61. cognite/neat/_data_model/models/dms/_view_property.py +235 -0
  62. cognite/neat/_data_model/models/dms/_views.py +216 -0
  63. cognite/neat/_data_model/models/entities/__init__.py +50 -0
  64. cognite/neat/_data_model/models/entities/_base.py +101 -0
  65. cognite/neat/_data_model/models/entities/_constants.py +22 -0
  66. cognite/neat/_data_model/models/entities/_data_types.py +144 -0
  67. cognite/neat/_data_model/models/entities/_identifiers.py +61 -0
  68. cognite/neat/_data_model/models/entities/_parser.py +226 -0
  69. cognite/neat/_data_model/validation/dms/__init__.py +75 -0
  70. cognite/neat/_data_model/validation/dms/_ai_readiness.py +381 -0
  71. cognite/neat/_data_model/validation/dms/_base.py +25 -0
  72. cognite/neat/_data_model/validation/dms/_connections.py +681 -0
  73. cognite/neat/_data_model/validation/dms/_consistency.py +58 -0
  74. cognite/neat/_data_model/validation/dms/_containers.py +199 -0
  75. cognite/neat/_data_model/validation/dms/_limits.py +368 -0
  76. cognite/neat/_data_model/validation/dms/_orchestrator.py +70 -0
  77. cognite/neat/_data_model/validation/dms/_views.py +164 -0
  78. cognite/neat/_exceptions.py +68 -0
  79. cognite/neat/_issues.py +68 -0
  80. cognite/neat/_session/__init__.py +3 -0
  81. cognite/neat/_session/_html/_render.py +30 -0
  82. cognite/neat/_session/_html/static/__init__.py +8 -0
  83. cognite/neat/_session/_html/static/deployment.css +476 -0
  84. cognite/neat/_session/_html/static/deployment.js +181 -0
  85. cognite/neat/_session/_html/static/issues.css +211 -0
  86. cognite/neat/_session/_html/static/issues.js +168 -0
  87. cognite/neat/_session/_html/static/shared.css +186 -0
  88. cognite/neat/_session/_html/templates/__init__.py +4 -0
  89. cognite/neat/_session/_html/templates/deployment.html +80 -0
  90. cognite/neat/_session/_html/templates/issues.html +45 -0
  91. cognite/neat/_session/_issues.py +81 -0
  92. cognite/neat/_session/_physical.py +294 -0
  93. cognite/neat/_session/_result/__init__.py +3 -0
  94. cognite/neat/_session/_result/_deployment/__init__.py +0 -0
  95. cognite/neat/_session/_result/_deployment/_physical/__init__.py +0 -0
  96. cognite/neat/_session/_result/_deployment/_physical/_changes.py +196 -0
  97. cognite/neat/_session/_result/_deployment/_physical/_statistics.py +180 -0
  98. cognite/neat/_session/_result/_deployment/_physical/serializer.py +35 -0
  99. cognite/neat/_session/_result/_result.py +31 -0
  100. cognite/neat/_session/_session.py +81 -0
  101. cognite/neat/_session/_usage_analytics/__init__.py +0 -0
  102. cognite/neat/_session/_usage_analytics/_collector.py +131 -0
  103. cognite/neat/_session/_usage_analytics/_constants.py +23 -0
  104. cognite/neat/_session/_usage_analytics/_storage.py +240 -0
  105. cognite/neat/_session/_wrappers.py +101 -0
  106. cognite/neat/_state_machine/__init__.py +10 -0
  107. cognite/neat/_state_machine/_base.py +37 -0
  108. cognite/neat/_state_machine/_states.py +52 -0
  109. cognite/neat/_store/__init__.py +3 -0
  110. cognite/neat/_store/_provenance.py +88 -0
  111. cognite/neat/_store/_store.py +220 -0
  112. cognite/neat/_utils/__init__.py +0 -0
  113. cognite/neat/_utils/_reader.py +194 -0
  114. cognite/neat/_utils/auxiliary.py +49 -0
  115. cognite/neat/_utils/collection.py +11 -0
  116. cognite/neat/_utils/http_client/__init__.py +39 -0
  117. cognite/neat/_utils/http_client/_client.py +245 -0
  118. cognite/neat/_utils/http_client/_config.py +19 -0
  119. cognite/neat/_utils/http_client/_data_classes.py +294 -0
  120. cognite/neat/_utils/http_client/_tracker.py +31 -0
  121. cognite/neat/_utils/repo.py +19 -0
  122. cognite/neat/_utils/text.py +71 -0
  123. cognite/neat/_utils/useful_types.py +37 -0
  124. cognite/neat/_utils/validation.py +154 -0
  125. cognite/neat/_v0/__init__.py +0 -0
  126. cognite/neat/_v0/core/__init__.py +0 -0
  127. cognite/neat/_v0/core/_client/_api/__init__.py +0 -0
  128. cognite/neat/{core → _v0/core}/_client/_api/data_modeling_loaders.py +8 -7
  129. cognite/neat/{core → _v0/core}/_client/_api/neat_instances.py +5 -5
  130. cognite/neat/{core → _v0/core}/_client/_api/schema.py +5 -5
  131. cognite/neat/{core → _v0/core}/_client/_api/statistics.py +3 -3
  132. cognite/neat/{core → _v0/core}/_client/_api_client.py +1 -1
  133. cognite/neat/_v0/core/_client/data_classes/__init__.py +0 -0
  134. cognite/neat/{core → _v0/core}/_client/data_classes/schema.py +4 -4
  135. cognite/neat/{core → _v0/core}/_client/testing.py +1 -1
  136. cognite/neat/{core → _v0/core}/_constants.py +5 -3
  137. cognite/neat/_v0/core/_data_model/__init__.py +0 -0
  138. cognite/neat/{core → _v0/core}/_data_model/_constants.py +7 -0
  139. cognite/neat/{core → _v0/core}/_data_model/_shared.py +4 -4
  140. cognite/neat/{core → _v0/core}/_data_model/analysis/_base.py +8 -8
  141. cognite/neat/{core → _v0/core}/_data_model/exporters/__init__.py +1 -2
  142. cognite/neat/{core → _v0/core}/_data_model/exporters/_base.py +7 -7
  143. cognite/neat/{core → _v0/core}/_data_model/exporters/_data_model2dms.py +9 -9
  144. cognite/neat/{core → _v0/core}/_data_model/exporters/_data_model2excel.py +12 -12
  145. cognite/neat/{core → _v0/core}/_data_model/exporters/_data_model2instance_template.py +4 -4
  146. cognite/neat/{core/_data_model/exporters/_data_model2ontology.py → _v0/core/_data_model/exporters/_data_model2semantic_model.py} +126 -116
  147. cognite/neat/{core → _v0/core}/_data_model/exporters/_data_model2yaml.py +1 -1
  148. cognite/neat/{core → _v0/core}/_data_model/importers/_base.py +5 -5
  149. cognite/neat/{core → _v0/core}/_data_model/importers/_base_file_reader.py +2 -2
  150. cognite/neat/{core → _v0/core}/_data_model/importers/_dict2data_model.py +5 -5
  151. cognite/neat/{core → _v0/core}/_data_model/importers/_dms2data_model.py +18 -15
  152. cognite/neat/{core → _v0/core}/_data_model/importers/_graph2data_model.py +12 -12
  153. cognite/neat/{core → _v0/core}/_data_model/importers/_rdf/_base.py +12 -12
  154. cognite/neat/{core → _v0/core}/_data_model/importers/_rdf/_inference2rdata_model.py +14 -14
  155. cognite/neat/{core → _v0/core}/_data_model/importers/_rdf/_owl2data_model.py +41 -21
  156. cognite/neat/{core → _v0/core}/_data_model/importers/_rdf/_shared.py +9 -9
  157. cognite/neat/{core → _v0/core}/_data_model/importers/_spreadsheet2data_model.py +92 -12
  158. cognite/neat/{core → _v0/core}/_data_model/models/__init__.py +3 -3
  159. cognite/neat/{core → _v0/core}/_data_model/models/_base_verified.py +5 -5
  160. cognite/neat/{core → _v0/core}/_data_model/models/_import_contexts.py +1 -1
  161. cognite/neat/{core → _v0/core}/_data_model/models/_types.py +5 -5
  162. cognite/neat/{core → _v0/core}/_data_model/models/conceptual/_unverified.py +16 -10
  163. cognite/neat/{core → _v0/core}/_data_model/models/conceptual/_validation.py +12 -12
  164. cognite/neat/{core → _v0/core}/_data_model/models/conceptual/_verified.py +9 -9
  165. cognite/neat/{core → _v0/core}/_data_model/models/data_types.py +14 -4
  166. cognite/neat/{core → _v0/core}/_data_model/models/entities/__init__.py +6 -0
  167. cognite/neat/_v0/core/_data_model/models/entities/_loaders.py +155 -0
  168. cognite/neat/{core → _v0/core}/_data_model/models/entities/_multi_value.py +2 -2
  169. cognite/neat/_v0/core/_data_model/models/entities/_restrictions.py +230 -0
  170. cognite/neat/{core → _v0/core}/_data_model/models/entities/_single_value.py +121 -16
  171. cognite/neat/{core → _v0/core}/_data_model/models/entities/_types.py +10 -0
  172. cognite/neat/{core → _v0/core}/_data_model/models/mapping/_classic2core.py +5 -5
  173. cognite/neat/{core → _v0/core}/_data_model/models/physical/__init__.py +1 -1
  174. cognite/neat/{core → _v0/core}/_data_model/models/physical/_exporter.py +26 -19
  175. cognite/neat/{core → _v0/core}/_data_model/models/physical/_unverified.py +133 -37
  176. cognite/neat/{core → _v0/core}/_data_model/models/physical/_validation.py +24 -20
  177. cognite/neat/{core → _v0/core}/_data_model/models/physical/_verified.py +95 -24
  178. cognite/neat/{core → _v0/core}/_data_model/transformers/_base.py +4 -4
  179. cognite/neat/{core → _v0/core}/_data_model/transformers/_converters.py +35 -28
  180. cognite/neat/{core → _v0/core}/_data_model/transformers/_mapping.py +7 -7
  181. cognite/neat/{core → _v0/core}/_data_model/transformers/_union_conceptual.py +5 -5
  182. cognite/neat/{core → _v0/core}/_data_model/transformers/_verification.py +7 -7
  183. cognite/neat/_v0/core/_instances/__init__.py +0 -0
  184. cognite/neat/{core → _v0/core}/_instances/_tracking/base.py +1 -1
  185. cognite/neat/{core → _v0/core}/_instances/_tracking/log.py +1 -1
  186. cognite/neat/{core → _v0/core}/_instances/extractors/__init__.py +3 -2
  187. cognite/neat/{core → _v0/core}/_instances/extractors/_base.py +6 -6
  188. cognite/neat/_v0/core/_instances/extractors/_classic_cdf/__init__.py +0 -0
  189. cognite/neat/{core → _v0/core}/_instances/extractors/_classic_cdf/_base.py +7 -7
  190. cognite/neat/{core → _v0/core}/_instances/extractors/_classic_cdf/_classic.py +12 -12
  191. cognite/neat/{core → _v0/core}/_instances/extractors/_classic_cdf/_relationships.py +3 -3
  192. cognite/neat/{core → _v0/core}/_instances/extractors/_classic_cdf/_sequences.py +2 -2
  193. cognite/neat/{core → _v0/core}/_instances/extractors/_dict.py +6 -3
  194. cognite/neat/{core → _v0/core}/_instances/extractors/_dms.py +6 -6
  195. cognite/neat/{core → _v0/core}/_instances/extractors/_dms_graph.py +11 -11
  196. cognite/neat/{core → _v0/core}/_instances/extractors/_mock_graph_generator.py +10 -10
  197. cognite/neat/{core → _v0/core}/_instances/extractors/_raw.py +3 -3
  198. cognite/neat/{core → _v0/core}/_instances/extractors/_rdf_file.py +7 -7
  199. cognite/neat/{core → _v0/core}/_instances/loaders/_base.py +5 -5
  200. cognite/neat/{core → _v0/core}/_instances/loaders/_rdf2dms.py +17 -17
  201. cognite/neat/{core → _v0/core}/_instances/loaders/_rdf_to_instance_space.py +11 -11
  202. cognite/neat/{core → _v0/core}/_instances/queries/_select.py +29 -3
  203. cognite/neat/{core → _v0/core}/_instances/queries/_update.py +1 -1
  204. cognite/neat/{core → _v0/core}/_instances/transformers/_base.py +4 -4
  205. cognite/neat/{core → _v0/core}/_instances/transformers/_classic_cdf.py +6 -6
  206. cognite/neat/{core → _v0/core}/_instances/transformers/_prune_graph.py +4 -4
  207. cognite/neat/{core → _v0/core}/_instances/transformers/_rdfpath.py +1 -1
  208. cognite/neat/{core → _v0/core}/_instances/transformers/_value_type.py +4 -4
  209. cognite/neat/{core → _v0/core}/_issues/_base.py +5 -5
  210. cognite/neat/{core → _v0/core}/_issues/_contextmanagers.py +1 -1
  211. cognite/neat/{core → _v0/core}/_issues/_factory.py +3 -3
  212. cognite/neat/{core → _v0/core}/_issues/errors/__init__.py +1 -1
  213. cognite/neat/{core → _v0/core}/_issues/errors/_external.py +1 -1
  214. cognite/neat/{core → _v0/core}/_issues/errors/_general.py +1 -1
  215. cognite/neat/{core → _v0/core}/_issues/errors/_properties.py +1 -1
  216. cognite/neat/{core → _v0/core}/_issues/errors/_resources.py +2 -2
  217. cognite/neat/{core → _v0/core}/_issues/errors/_wrapper.py +7 -3
  218. cognite/neat/{core → _v0/core}/_issues/warnings/__init__.py +1 -1
  219. cognite/neat/{core → _v0/core}/_issues/warnings/_external.py +1 -1
  220. cognite/neat/{core → _v0/core}/_issues/warnings/_general.py +1 -1
  221. cognite/neat/{core → _v0/core}/_issues/warnings/_models.py +2 -2
  222. cognite/neat/{core → _v0/core}/_issues/warnings/_properties.py +2 -2
  223. cognite/neat/{core → _v0/core}/_issues/warnings/_resources.py +1 -1
  224. cognite/neat/{core → _v0/core}/_issues/warnings/user_modeling.py +1 -1
  225. cognite/neat/{core → _v0/core}/_store/_data_model.py +12 -12
  226. cognite/neat/{core → _v0/core}/_store/_instance.py +43 -10
  227. cognite/neat/{core → _v0/core}/_store/_provenance.py +3 -3
  228. cognite/neat/{core → _v0/core}/_store/exceptions.py +4 -4
  229. cognite/neat/_v0/core/_utils/__init__.py +0 -0
  230. cognite/neat/{core → _v0/core}/_utils/auth.py +22 -12
  231. cognite/neat/{core → _v0/core}/_utils/auxiliary.py +1 -1
  232. cognite/neat/{core → _v0/core}/_utils/collection_.py +2 -2
  233. cognite/neat/{core → _v0/core}/_utils/graph_transformations_report.py +1 -1
  234. cognite/neat/{core → _v0/core}/_utils/rdf_.py +1 -1
  235. cognite/neat/{core → _v0/core}/_utils/reader/_base.py +1 -1
  236. cognite/neat/{core → _v0/core}/_utils/spreadsheet.py +18 -4
  237. cognite/neat/{core → _v0/core}/_utils/text.py +1 -1
  238. cognite/neat/{core → _v0/core}/_utils/upload.py +3 -3
  239. cognite/neat/{session → _v0}/engine/_load.py +1 -1
  240. cognite/neat/_v0/plugins/__init__.py +4 -0
  241. cognite/neat/_v0/plugins/_base.py +9 -0
  242. cognite/neat/_v0/plugins/_data_model.py +48 -0
  243. cognite/neat/{plugins → _v0/plugins}/_issues.py +1 -1
  244. cognite/neat/{plugins → _v0/plugins}/_manager.py +7 -16
  245. cognite/neat/{session → _v0/session}/_base.py +13 -15
  246. cognite/neat/{session → _v0/session}/_collector.py +1 -1
  247. cognite/neat/_v0/session/_diff.py +51 -0
  248. cognite/neat/{session → _v0/session}/_drop.py +3 -3
  249. cognite/neat/{session → _v0/session}/_explore.py +2 -2
  250. cognite/neat/{session → _v0/session}/_fix.py +2 -2
  251. cognite/neat/{session → _v0/session}/_inspect.py +3 -3
  252. cognite/neat/{session → _v0/session}/_mapping.py +3 -3
  253. cognite/neat/{session → _v0/session}/_plugin.py +4 -5
  254. cognite/neat/{session → _v0/session}/_prepare.py +8 -8
  255. cognite/neat/{session → _v0/session}/_read.py +34 -21
  256. cognite/neat/{session → _v0/session}/_set.py +8 -8
  257. cognite/neat/{session → _v0/session}/_show.py +5 -5
  258. cognite/neat/{session → _v0/session}/_state.py +10 -10
  259. cognite/neat/{session → _v0/session}/_subset.py +4 -4
  260. cognite/neat/{session → _v0/session}/_template.py +11 -11
  261. cognite/neat/{session → _v0/session}/_to.py +12 -12
  262. cognite/neat/{session → _v0/session}/_wizard.py +1 -1
  263. cognite/neat/{session → _v0/session}/exceptions.py +5 -5
  264. cognite/neat/_version.py +1 -1
  265. cognite/neat/legacy.py +6 -0
  266. cognite_neat-1.0.22.dist-info/METADATA +123 -0
  267. cognite_neat-1.0.22.dist-info/RECORD +329 -0
  268. cognite_neat-1.0.22.dist-info/WHEEL +4 -0
  269. cognite/neat/core/_data_model/models/entities/_loaders.py +0 -75
  270. cognite/neat/plugins/__init__.py +0 -3
  271. cognite/neat/plugins/data_model/importers/__init__.py +0 -5
  272. cognite/neat/plugins/data_model/importers/_base.py +0 -28
  273. cognite/neat/session/_session/_data_model/__init__.py +0 -3
  274. cognite/neat/session/_session/_data_model/_read.py +0 -193
  275. cognite/neat/session/_session/_data_model/_routes.py +0 -45
  276. cognite/neat/session/_session/_data_model/_show.py +0 -147
  277. cognite/neat/session/_session/_data_model/_write.py +0 -335
  278. cognite_neat-0.123.26.dist-info/METADATA +0 -144
  279. cognite_neat-0.123.26.dist-info/RECORD +0 -201
  280. cognite_neat-0.123.26.dist-info/WHEEL +0 -4
  281. cognite_neat-0.123.26.dist-info/licenses/LICENSE +0 -201
  282. /cognite/neat/{core → _client/init}/__init__.py +0 -0
  283. /cognite/neat/{core/_client/_api → _data_model}/__init__.py +0 -0
  284. /cognite/neat/{core/_client/data_classes → _data_model/deployer}/__init__.py +0 -0
  285. /cognite/neat/{core/_data_model → _data_model/exporters/_table_exporter}/__init__.py +0 -0
  286. /cognite/neat/{core/_instances → _data_model/importers/_table_importer}/__init__.py +0 -0
  287. /cognite/neat/{core/_instances/extractors/_classic_cdf → _data_model/models}/__init__.py +0 -0
  288. /cognite/neat/{core/_utils → _data_model/models/conceptual}/__init__.py +0 -0
  289. /cognite/neat/{plugins/data_model → _data_model/validation}/__init__.py +0 -0
  290. /cognite/neat/{session/_session → _session/_html}/__init__.py +0 -0
  291. /cognite/neat/{core → _v0/core}/_client/__init__.py +0 -0
  292. /cognite/neat/{core → _v0/core}/_client/data_classes/data_modeling.py +0 -0
  293. /cognite/neat/{core → _v0/core}/_client/data_classes/neat_sequence.py +0 -0
  294. /cognite/neat/{core → _v0/core}/_client/data_classes/statistics.py +0 -0
  295. /cognite/neat/{core → _v0/core}/_config.py +0 -0
  296. /cognite/neat/{core → _v0/core}/_data_model/analysis/__init__.py +0 -0
  297. /cognite/neat/{core → _v0/core}/_data_model/catalog/__init__.py +0 -0
  298. /cognite/neat/{core → _v0/core}/_data_model/catalog/classic_model.xlsx +0 -0
  299. /cognite/neat/{core → _v0/core}/_data_model/catalog/conceptual-imf-data-model.xlsx +0 -0
  300. /cognite/neat/{core → _v0/core}/_data_model/catalog/hello_world_pump.xlsx +0 -0
  301. /cognite/neat/{core → _v0/core}/_data_model/importers/__init__.py +0 -0
  302. /cognite/neat/{core → _v0/core}/_data_model/importers/_rdf/__init__.py +0 -0
  303. /cognite/neat/{core → _v0/core}/_data_model/models/_base_unverified.py +0 -0
  304. /cognite/neat/{core → _v0/core}/_data_model/models/conceptual/__init__.py +0 -0
  305. /cognite/neat/{core → _v0/core}/_data_model/models/entities/_constants.py +0 -0
  306. /cognite/neat/{core → _v0/core}/_data_model/models/entities/_wrapped.py +0 -0
  307. /cognite/neat/{core → _v0/core}/_data_model/models/mapping/__init__.py +0 -0
  308. /cognite/neat/{core → _v0/core}/_data_model/models/mapping/_classic2core.yaml +0 -0
  309. /cognite/neat/{core → _v0/core}/_data_model/transformers/__init__.py +0 -0
  310. /cognite/neat/{core → _v0/core}/_instances/_shared.py +0 -0
  311. /cognite/neat/{core → _v0/core}/_instances/_tracking/__init__.py +0 -0
  312. /cognite/neat/{core → _v0/core}/_instances/examples/Knowledge-Graph-Nordic44-dirty.xml +0 -0
  313. /cognite/neat/{core → _v0/core}/_instances/examples/Knowledge-Graph-Nordic44.xml +0 -0
  314. /cognite/neat/{core → _v0/core}/_instances/examples/__init__.py +0 -0
  315. /cognite/neat/{core → _v0/core}/_instances/examples/skos-capturing-sheet-wind-topics.xlsx +0 -0
  316. /cognite/neat/{core → _v0/core}/_instances/extractors/_classic_cdf/_assets.py +0 -0
  317. /cognite/neat/{core → _v0/core}/_instances/extractors/_classic_cdf/_data_sets.py +0 -0
  318. /cognite/neat/{core → _v0/core}/_instances/extractors/_classic_cdf/_events.py +0 -0
  319. /cognite/neat/{core → _v0/core}/_instances/extractors/_classic_cdf/_files.py +0 -0
  320. /cognite/neat/{core → _v0/core}/_instances/extractors/_classic_cdf/_labels.py +0 -0
  321. /cognite/neat/{core → _v0/core}/_instances/extractors/_classic_cdf/_timeseries.py +0 -0
  322. /cognite/neat/{core → _v0/core}/_instances/loaders/__init__.py +0 -0
  323. /cognite/neat/{core → _v0/core}/_instances/queries/__init__.py +0 -0
  324. /cognite/neat/{core → _v0/core}/_instances/queries/_base.py +0 -0
  325. /cognite/neat/{core → _v0/core}/_instances/queries/_queries.py +0 -0
  326. /cognite/neat/{core → _v0/core}/_instances/transformers/__init__.py +0 -0
  327. /cognite/neat/{core → _v0/core}/_issues/__init__.py +0 -0
  328. /cognite/neat/{core → _v0/core}/_issues/formatters.py +0 -0
  329. /cognite/neat/{core → _v0/core}/_shared.py +0 -0
  330. /cognite/neat/{core → _v0/core}/_store/__init__.py +0 -0
  331. /cognite/neat/{core → _v0/core}/_utils/io_.py +0 -0
  332. /cognite/neat/{core → _v0/core}/_utils/reader/__init__.py +0 -0
  333. /cognite/neat/{core → _v0/core}/_utils/tarjan.py +0 -0
  334. /cognite/neat/{core → _v0/core}/_utils/time_.py +0 -0
  335. /cognite/neat/{core → _v0/core}/_utils/xml_.py +0 -0
  336. /cognite/neat/{session → _v0}/engine/__init__.py +0 -0
  337. /cognite/neat/{session → _v0}/engine/_import.py +0 -0
  338. /cognite/neat/{session → _v0}/engine/_interface.py +0 -0
  339. /cognite/neat/{session → _v0/session}/__init__.py +0 -0
  340. /cognite/neat/{session → _v0/session}/_experimental.py +0 -0
  341. /cognite/neat/{session → _v0/session}/_state/README.md +0 -0
@@ -0,0 +1,431 @@
1
+ from collections.abc import Callable, Mapping, Sequence
2
+ from dataclasses import dataclass
3
+ from functools import partial
4
+ from typing import cast
5
+
6
+ from cognite.neat._client import NeatClient
7
+ from cognite.neat._data_model._constants import COGNITE_SPACES
8
+ from cognite.neat._data_model._shared import OnSuccessResultProducer
9
+ from cognite.neat._data_model._snapshot import SchemaSnapshot
10
+ from cognite.neat._data_model.models.dms import (
11
+ ContainerRequest,
12
+ DataModelBody,
13
+ RequestSchema,
14
+ T_DataModelResource,
15
+ T_ResourceId,
16
+ )
17
+ from cognite.neat._utils.collection import chunker_sequence
18
+ from cognite.neat._utils.http_client import (
19
+ FailedRequestItems,
20
+ FailedResponseItems,
21
+ ItemIDBody,
22
+ ItemsRequest,
23
+ SuccessResponseItems,
24
+ )
25
+ from cognite.neat._utils.http_client._data_classes import APIResponse
26
+ from cognite.neat._utils.useful_types import ModusOperandi, T_Reference
27
+
28
+ from ._differ import ItemDiffer
29
+ from ._differ_container import ContainerDiffer
30
+ from ._differ_data_model import DataModelDiffer
31
+ from ._differ_space import SpaceDiffer
32
+ from ._differ_view import ViewDiffer
33
+ from .data_classes import (
34
+ AddedField,
35
+ AppliedChanges,
36
+ ChangedFieldResult,
37
+ ContainerDeploymentPlan,
38
+ DataModelEndpoint,
39
+ DeploymentResult,
40
+ FieldChange,
41
+ FieldChanges,
42
+ HTTPChangeResult,
43
+ NoOpChangeResult,
44
+ RemovedField,
45
+ ResourceChange,
46
+ ResourceDeploymentPlan,
47
+ ResourceDeploymentPlanList,
48
+ SeverityType,
49
+ )
50
+
51
+
52
+ @dataclass
53
+ class DeploymentOptions:
54
+ """Configuration options for deployment.
55
+
56
+ Attributes:
57
+ dry_run (bool): If True, the deployment will be simulated without applying changes. Defaults to True.
58
+ auto_rollback (bool): If True, automatically roll back changes if deployment fails. Defaults to True.
59
+ max_severity (SeverityType): Maximum allowed severity of changes to proceed with deployment.
60
+ Defaults to SeverityType.SAFE.
61
+ modus_operandi (ModusOperandi): Deployment mode. If "additive", only add/modify resources
62
+ specified in the data model. If "rebuild", remove resources not present in the data model.
63
+ Defaults to "additive".
64
+ """
65
+
66
+ dry_run: bool = True
67
+ auto_rollback: bool = True
68
+ drop_data: bool = False
69
+ max_severity: SeverityType = SeverityType.WARNING
70
+ modus_operandi: ModusOperandi = "additive"
71
+
72
+
73
+ class SchemaDeployer(OnSuccessResultProducer):
74
+ INDEX_DELETE_BATCH_SIZE = 10
75
+ CONSTRAINT_DELETE_BATCH_SIZE = 10
76
+
77
+ def __init__(self, client: NeatClient, options: DeploymentOptions | None = None) -> None:
78
+ super().__init__()
79
+ self.client: NeatClient = client
80
+ self.options: DeploymentOptions = options or DeploymentOptions()
81
+
82
+ def run(self, data_model: RequestSchema) -> None:
83
+ if self._results is not None:
84
+ raise RuntimeError("SchemaDeployer has already been run.")
85
+ self._results = self.deploy(data_model)
86
+
87
+ def deploy(self, data_model: RequestSchema) -> DeploymentResult:
88
+ # Step 1: Fetch current CDF state
89
+ snapshot = SchemaSnapshot.fetch_cdf_data_model(self.client, data_model)
90
+
91
+ # Step 2: Create deployment plan by comparing local vs cdf
92
+ plan = self.create_deployment_plan(snapshot, data_model)
93
+
94
+ # Step 3: Adjust plan based on modus operandi
95
+ if self.options.modus_operandi == "additive":
96
+ # Filter out deletions and removals in additive mode
97
+ plan = plan.consolidate_changes()
98
+ elif self.options.modus_operandi == "rebuild":
99
+ # Breaking changes are forced by deleting and recreating resources
100
+ # Containers are treated as additive unless drop_data is specified
101
+ plan = plan.force_changes(self.options.drop_data)
102
+ else:
103
+ raise NotImplementedError(f"Unsupported modus operandi: {self.options.modus_operandi!r}")
104
+
105
+ if not self.should_proceed_to_deploy(plan):
106
+ # Step 4: Check if deployment should proceed
107
+ return DeploymentResult(status="failure", plan=list(plan), snapshot=snapshot)
108
+ elif self.options.dry_run:
109
+ # Step 5: If dry-run, return plan without applying
110
+ return DeploymentResult(status="pending", plan=list(plan), snapshot=snapshot)
111
+
112
+ # Step 6: Apply changes
113
+ changes = self.apply_changes(plan)
114
+
115
+ # Step 7: Rollback if failed and auto_rollback is enabled
116
+ if not changes.is_success and self.options.auto_rollback:
117
+ recovery = self.apply_changes(changes.as_recovery_plan())
118
+ return DeploymentResult(
119
+ status="recovered" if recovery.is_success else "recovery_failed",
120
+ plan=list(plan),
121
+ snapshot=snapshot,
122
+ responses=changes,
123
+ recovery=recovery,
124
+ )
125
+ return DeploymentResult(
126
+ status="success" if changes.is_success else "partial", plan=list(plan), snapshot=snapshot, responses=changes
127
+ )
128
+
129
+ def create_deployment_plan(self, snapshot: SchemaSnapshot, data_model: RequestSchema) -> ResourceDeploymentPlanList:
130
+ return ResourceDeploymentPlanList(
131
+ [
132
+ self._create_resource_plan(
133
+ snapshot.spaces,
134
+ data_model.spaces,
135
+ "spaces",
136
+ SpaceDiffer(),
137
+ skip_criteria=partial(self._skip_resource, model_space=data_model.data_model.space),
138
+ ),
139
+ self._create_resource_plan(
140
+ snapshot.containers,
141
+ data_model.containers,
142
+ "containers",
143
+ ContainerDiffer(),
144
+ ContainerDeploymentPlan,
145
+ skip_criteria=partial(self._skip_resource, model_space=data_model.data_model.space),
146
+ ),
147
+ self._create_resource_plan(
148
+ snapshot.views,
149
+ data_model.views,
150
+ "views",
151
+ ViewDiffer(
152
+ current_container_map=snapshot.containers,
153
+ new_container_map={container.as_reference(): container for container in data_model.containers},
154
+ ),
155
+ skip_criteria=partial(self._skip_resource, model_space=data_model.data_model.space),
156
+ ),
157
+ self._create_resource_plan(
158
+ snapshot.data_model,
159
+ [data_model.data_model],
160
+ "datamodels",
161
+ DataModelDiffer(),
162
+ skip_criteria=partial(self._skip_resource, model_space=data_model.data_model.space),
163
+ ),
164
+ ]
165
+ )
166
+
167
+ def _create_resource_plan(
168
+ self,
169
+ current_resources: dict[T_ResourceId, T_DataModelResource],
170
+ new_resources: list[T_DataModelResource],
171
+ endpoint: DataModelEndpoint,
172
+ differ: ItemDiffer[T_DataModelResource],
173
+ plan_type: type[ResourceDeploymentPlan[T_ResourceId, T_DataModelResource]] = ResourceDeploymentPlan,
174
+ skip_criteria: Callable[[T_ResourceId], str | None] | None = None,
175
+ ) -> ResourceDeploymentPlan[T_ResourceId, T_DataModelResource]:
176
+ resources: list[ResourceChange[T_ResourceId, T_DataModelResource]] = []
177
+ for new_resource in new_resources:
178
+ # We know that .as_reference() will return T_ResourceId
179
+ ref = cast(T_ResourceId, new_resource.as_reference())
180
+ if skip_criteria is not None and (reason := skip_criteria(ref)):
181
+ resources.append(ResourceChange(resource_id=ref, new_value=None, current_value=None, message=reason))
182
+ continue
183
+ if ref not in current_resources:
184
+ resources.append(ResourceChange(resource_id=ref, new_value=new_resource))
185
+ continue
186
+ current_resource = current_resources[ref]
187
+ diffs = differ.diff(current_resource, new_resource)
188
+ if (
189
+ isinstance(current_resource, ContainerRequest)
190
+ and isinstance(new_resource, ContainerRequest)
191
+ and self.options.modus_operandi == "additive"
192
+ ):
193
+ # In additive mode, changes to constraints and indexes require removal and re-adding
194
+ # In rebuild mode, all changes are forced via deletion and re-adding
195
+ diffs = self.remove_readd_modified_indexes_and_constraints(diffs, current_resource, new_resource)
196
+ resources.append(
197
+ ResourceChange(resource_id=ref, new_value=new_resource, current_value=current_resource, changes=diffs)
198
+ )
199
+
200
+ return plan_type(endpoint=endpoint, resources=resources)
201
+
202
+ @classmethod
203
+ def remove_readd_modified_indexes_and_constraints(
204
+ cls, diffs: list[FieldChange], current_resource: ContainerRequest, new_resource: ContainerRequest
205
+ ) -> list[FieldChange]:
206
+ """Constraints and indexes cannot be modified directly; they must be removed and re-added.
207
+
208
+ Args:
209
+ diffs: The list of field changes detected by the differ.
210
+ current_resource: The current state of the container.
211
+ new_resource: The desired state of the container.
212
+ Returns:
213
+ A modified list of field changes with constraints and indexes handled appropriately.
214
+ """
215
+ modified_diffs: list[FieldChange] = []
216
+ for diff in diffs:
217
+ if (diff.field_path.startswith("constraints") or diff.field_path.startswith("indexes")) and isinstance(
218
+ diff, FieldChanges
219
+ ):
220
+ if "." not in diff.field_path:
221
+ # Should not happen, but just in case
222
+ raise RuntimeError("Bug in Neat. Malformed field path for constraint/index change.")
223
+ # Field type is either "constraints" or "indexes"
224
+ field_type, identifier, *_ = diff.field_path.split(".", maxsplit=2)
225
+ # Mark for removal
226
+ modified_diffs.append(
227
+ RemovedField(
228
+ field_path=f"{field_type}.{identifier}",
229
+ item_severity=SeverityType.WARNING,
230
+ current_value=getattr(current_resource, field_type)[identifier],
231
+ )
232
+ )
233
+ # Mark for addition
234
+ modified_diffs.append(
235
+ AddedField(
236
+ field_path=f"{field_type}.{identifier}",
237
+ item_severity=SeverityType.SAFE,
238
+ new_value=getattr(new_resource, field_type)[identifier],
239
+ )
240
+ )
241
+ else:
242
+ modified_diffs.append(diff)
243
+ return modified_diffs
244
+
245
+ @classmethod
246
+ def _skip_resource(cls, resource_id: T_ResourceId, model_space: str) -> str | None:
247
+ """Checks if a resource should be skipped based on its space.
248
+
249
+ Args:
250
+ resource_id: The ID of the resource to check.
251
+ model_space: The space of the data model.
252
+
253
+ Returns:
254
+ A reason for skipping if the resource space does not match the model space, otherwise None.
255
+ """
256
+ if resource_id.space in COGNITE_SPACES:
257
+ return f"Skipping resource as it is in the reserved Cognite space '{resource_id.space}'."
258
+ elif resource_id.space != model_space:
259
+ return (
260
+ f"Skipping resource as it is in the space '{resource_id.space}'"
261
+ f" not matching data model space '{model_space}'."
262
+ )
263
+ return None
264
+
265
+ def should_proceed_to_deploy(self, plan: Sequence[ResourceDeploymentPlan]) -> bool:
266
+ max_severity_in_plan = SeverityType.max_severity(
267
+ [change.severity for resource_plan in plan for change in resource_plan.resources],
268
+ default=SeverityType.SAFE,
269
+ )
270
+ return max_severity_in_plan.value <= self.options.max_severity.value
271
+
272
+ def apply_changes(self, plan: Sequence[ResourceDeploymentPlan]) -> AppliedChanges:
273
+ """Applies the given deployment plan to CDF by making the necessary API calls.
274
+
275
+ Args:
276
+ plan (list[ResourceDeploymentPlan]): The deployment plan to apply.
277
+
278
+ Returns:
279
+ AppliedChanges: The result of applying the changes.
280
+ """
281
+ applied_changes = AppliedChanges()
282
+ # If any HTTP request fails, the skip_message will be set and subsequent operations will be skipped
283
+ failure_message: str | None = None
284
+ for resource in reversed(plan):
285
+ if failure_message is None:
286
+ deletions = self._delete_items(resource)
287
+ applied_changes.deletions.extend(deletions)
288
+ if any(not deletion.is_success for deletion in deletions):
289
+ failure_message = f"Skipping due to {resource.endpoint} deletions failing."
290
+ else:
291
+ applied_changes.skipped.extend(
292
+ [
293
+ NoOpChangeResult(endpoint=resource.endpoint, change=change, reason=failure_message)
294
+ for change in resource.to_delete
295
+ ]
296
+ )
297
+
298
+ for resource in plan:
299
+ if failure_message is None:
300
+ if isinstance(resource, ContainerDeploymentPlan):
301
+ # Note that we continue to deploy even if removing constraints/indexes fail,
302
+ # as the creation/update of views and data models will still succeed.
303
+ applied_changes.changed_fields.extend(self._remove_container_constraints(resource))
304
+ applied_changes.changed_fields.extend(self._remove_container_indexes(resource))
305
+
306
+ creations, updated = self._upsert_items(resource)
307
+ applied_changes.created.extend(creations)
308
+ applied_changes.updated.extend(updated)
309
+ if any(not change.is_success for change in creations + updated):
310
+ failure_message = f"Skipping due to {resource.endpoint} upsert failing."
311
+ else:
312
+ applied_changes.skipped.extend(
313
+ [
314
+ NoOpChangeResult(endpoint=resource.endpoint, change=change, reason=failure_message)
315
+ for change in resource.to_upsert
316
+ ]
317
+ )
318
+
319
+ applied_changes.unchanged.extend(
320
+ [
321
+ NoOpChangeResult(endpoint=resource.endpoint, change=change, reason="No changes detected.")
322
+ for change in resource.unchanged
323
+ ]
324
+ )
325
+ applied_changes.skipped.extend(
326
+ [
327
+ NoOpChangeResult(endpoint=resource.endpoint, change=change, reason=change.message or "Unknown")
328
+ for change in resource.skip
329
+ ]
330
+ )
331
+ return applied_changes
332
+
333
+ def _delete_items(self, resource: ResourceDeploymentPlan) -> list[HTTPChangeResult]:
334
+ to_delete_by_id = {change.resource_id: change for change in resource.to_delete}
335
+ if not to_delete_by_id:
336
+ return []
337
+ responses = self.client.http_client.request_with_retries(
338
+ ItemsRequest(
339
+ endpoint_url=self.client.config.create_api_url(f"/models/{resource.endpoint}/delete"),
340
+ method="POST",
341
+ body=ItemIDBody(items=list(to_delete_by_id.keys())),
342
+ )
343
+ )
344
+ return self._process_resource_responses(responses, to_delete_by_id, resource.endpoint)
345
+
346
+ def _upsert_items(self, resource: ResourceDeploymentPlan) -> tuple[list[HTTPChangeResult], list[HTTPChangeResult]]:
347
+ to_upsert = [
348
+ resource_change.new_value for resource_change in resource.to_upsert if resource_change.new_value is not None
349
+ ]
350
+ if not to_upsert:
351
+ return [], []
352
+ responses = self.client.http_client.request_with_retries(
353
+ ItemsRequest(
354
+ endpoint_url=self.client.config.create_api_url(f"/models/{resource.endpoint}"),
355
+ method="POST",
356
+ body=DataModelBody(items=to_upsert),
357
+ )
358
+ )
359
+ to_create_by_id = {rc.resource_id: rc for rc in resource.to_create}
360
+ create_result = self._process_resource_responses(responses, to_create_by_id, resource.endpoint)
361
+ to_update_by_id = {rc.resource_id: rc for rc in resource.to_update}
362
+ update_result = self._process_resource_responses(responses, to_update_by_id, resource.endpoint)
363
+ return create_result, update_result
364
+
365
+ def _remove_container_indexes(self, resource: ContainerDeploymentPlan) -> list[ChangedFieldResult]:
366
+ return self._remove_container_fields(
367
+ resource.indexes_to_remove,
368
+ "/models/containers/indexes/delete",
369
+ self.INDEX_DELETE_BATCH_SIZE,
370
+ )
371
+
372
+ def _remove_container_constraints(self, resource: ContainerDeploymentPlan) -> list[ChangedFieldResult]:
373
+ return self._remove_container_fields(
374
+ resource.constraints_to_remove,
375
+ "/models/containers/constraints/delete",
376
+ self.CONSTRAINT_DELETE_BATCH_SIZE,
377
+ )
378
+
379
+ def _remove_container_fields(
380
+ self,
381
+ fields_to_remove: Mapping[T_Reference, FieldChange],
382
+ endpoint: str,
383
+ batch_size: int,
384
+ ) -> list[ChangedFieldResult]:
385
+ if not fields_to_remove:
386
+ return []
387
+ results: list[ChangedFieldResult] = []
388
+ for batch in chunker_sequence(list(fields_to_remove.keys()), batch_size):
389
+ responses = self.client.http_client.request_with_retries(
390
+ ItemsRequest(
391
+ endpoint_url=self.client.config.create_api_url(endpoint),
392
+ method="POST",
393
+ body=ItemIDBody(items=batch),
394
+ )
395
+ )
396
+ results.extend(self._process_field_responses(responses, fields_to_remove))
397
+ return results
398
+
399
+ @classmethod
400
+ def _process_resource_responses(
401
+ cls, responses: APIResponse, change_by_id: dict[T_ResourceId, ResourceChange], endpoint: DataModelEndpoint
402
+ ) -> list[HTTPChangeResult]:
403
+ results: list[HTTPChangeResult] = []
404
+ for response in responses:
405
+ if isinstance(response, SuccessResponseItems | FailedResponseItems | FailedRequestItems):
406
+ for id in response.ids:
407
+ if id not in change_by_id:
408
+ continue
409
+ results.append(HTTPChangeResult(change=change_by_id[id], http_message=response, endpoint=endpoint))
410
+ else:
411
+ # This should never happen as we do a ItemsRequest should always return ItemMessage responses
412
+ raise ValueError("Bug in Neat. Got an unexpected response type.")
413
+ return results
414
+
415
+ @classmethod
416
+ def _process_field_responses(
417
+ cls, responses: APIResponse, change_by_id: Mapping[T_Reference, FieldChange]
418
+ ) -> list[ChangedFieldResult]:
419
+ results: list[ChangedFieldResult] = []
420
+ for response in responses:
421
+ if isinstance(response, SuccessResponseItems | FailedResponseItems | FailedRequestItems):
422
+ for id in response.ids:
423
+ if id not in change_by_id:
424
+ continue
425
+ results.append(
426
+ ChangedFieldResult(resource_id=id, field_change=change_by_id[id], http_message=response)
427
+ )
428
+ else:
429
+ # This should never happen as we do a ItemsRequest should always return ItemMessage responses
430
+ raise RuntimeError("Bug in Neat. Got an unexpected response type.")
431
+ return results
@@ -0,0 +1,15 @@
1
+ from ._api_exporter import DMSAPIExporter, DMSAPIJSONExporter, DMSAPIYAMLExporter
2
+ from ._base import DMSExporter, DMSFileExporter
3
+ from ._table_exporter.exporter import DMSExcelExporter, DMSTableExporter, DMSTableJSONExporter, DMSTableYamlExporter
4
+
5
+ __all__ = [
6
+ "DMSAPIExporter",
7
+ "DMSAPIJSONExporter",
8
+ "DMSAPIYAMLExporter",
9
+ "DMSExcelExporter",
10
+ "DMSExporter",
11
+ "DMSFileExporter",
12
+ "DMSTableExporter",
13
+ "DMSTableJSONExporter",
14
+ "DMSTableYamlExporter",
15
+ ]
@@ -0,0 +1,37 @@
1
+ from pathlib import Path
2
+
3
+ import yaml
4
+
5
+ from cognite.neat._data_model.exporters._base import DMSExporter, DMSFileExporter
6
+ from cognite.neat._data_model.models.dms import RequestSchema
7
+
8
+
9
+ class DMSAPIExporter(DMSExporter[RequestSchema]):
10
+ def export(self, data_model: RequestSchema) -> RequestSchema:
11
+ return data_model
12
+
13
+ def export_to_file(self, data_model: RequestSchema, file_path: Path) -> None:
14
+ raise RuntimeError(f"{type(self).__name__} does not support export_to_file method.")
15
+
16
+
17
+ class DMSAPIYAMLExporter(DMSAPIExporter, DMSFileExporter[RequestSchema]):
18
+ def export_to_file(self, data_model: RequestSchema, file_path: Path) -> None:
19
+ """Export the data model to a YAML file in API format."""
20
+ if file_path.suffix.lower() not in {".yaml", ".yml"}:
21
+ raise ValueError("The file path must have a .yaml or .yml extension.")
22
+
23
+ api_format = data_model.model_dump(mode="json", by_alias=True)
24
+ file_path.write_text(yaml.safe_dump(api_format, sort_keys=False), encoding=self.ENCODING, newline=self.NEW_LINE)
25
+
26
+
27
+ class DMSAPIJSONExporter(DMSAPIExporter, DMSFileExporter[RequestSchema]):
28
+ def export_to_file(self, data_model: RequestSchema, file_path: Path) -> None:
29
+ """Export the data model to a JSON file in API format."""
30
+ if file_path.suffix.lower() != ".json":
31
+ raise ValueError("The file path must have a .json extension.")
32
+
33
+ file_path.write_text(
34
+ data_model.model_dump_json(by_alias=True),
35
+ encoding=self.ENCODING,
36
+ newline=self.NEW_LINE,
37
+ )
@@ -0,0 +1,24 @@
1
+ from abc import ABC, abstractmethod
2
+ from pathlib import Path
3
+ from typing import Generic, TypeVar
4
+
5
+ from cognite.neat._data_model.models.dms import RequestSchema
6
+
7
+ T_Export = TypeVar("T_Export")
8
+
9
+
10
+ class DMSExporter(ABC, Generic[T_Export]):
11
+ """This is the base class for all DMS exporters."""
12
+
13
+ NEW_LINE = "\n"
14
+ ENCODING = "utf-8"
15
+
16
+ @abstractmethod
17
+ def export(self, data_model: RequestSchema) -> T_Export:
18
+ raise NotImplementedError()
19
+
20
+
21
+ class DMSFileExporter(DMSExporter[T_Export], ABC):
22
+ @abstractmethod
23
+ def export_to_file(self, data_model: RequestSchema, file_path: Path) -> None:
24
+ raise NotImplementedError()
@@ -0,0 +1,128 @@
1
+ import json
2
+ from pathlib import Path
3
+ from typing import cast
4
+
5
+ import yaml
6
+
7
+ from cognite.neat._data_model.exporters._base import DMSExporter, DMSFileExporter
8
+ from cognite.neat._data_model.importers._table_importer.data_classes import DMSProperty, TableDMS
9
+ from cognite.neat._data_model.models.dms import RequestSchema
10
+ from cognite.neat._utils.useful_types import DataModelTableType
11
+
12
+ from .workbook import WorkbookCreator, WorkbookOptions
13
+ from .writer import DMSTableWriter
14
+
15
+
16
+ class DMSTableExporter(DMSExporter[DataModelTableType]):
17
+ """Exports DMS to a table structure.
18
+
19
+ The tables can are expected to be a dictionary where the keys are the table names and the values
20
+ are lists of dictionaries representing the rows in the table.
21
+ """
22
+
23
+ class Sheets:
24
+ properties = cast(str, TableDMS.model_fields["properties"].validation_alias)
25
+
26
+ def __init__(self, exclude_none: bool = False, skip_properties_in_other_spaces: bool = True) -> None:
27
+ self._exclude_none = exclude_none
28
+ self._skip_properties_in_other_spaces = skip_properties_in_other_spaces
29
+
30
+ def export(self, data_model: RequestSchema) -> DataModelTableType:
31
+ model = data_model.data_model
32
+ tables = DMSTableWriter(model.space, model.version, self._skip_properties_in_other_spaces).write_tables(
33
+ data_model
34
+ )
35
+ exclude: set[str] = set()
36
+ if self._exclude_none:
37
+ if not tables.enum:
38
+ exclude.add("enum")
39
+ if not tables.nodes:
40
+ exclude.add("nodes")
41
+ if not tables.containers:
42
+ exclude.add("containers")
43
+
44
+ output = tables.model_dump(mode="json", by_alias=True, exclude_none=self._exclude_none, exclude=exclude)
45
+ # When we have exclude_none we only want to exclude none of optional properties, not required.
46
+ # Thus, we do the implementation below
47
+ required_properties = [
48
+ field_.serialization_alias for field_ in DMSProperty.model_fields.values() if field_.is_required()
49
+ ]
50
+ for row in output[self.Sheets.properties]:
51
+ for prop in required_properties:
52
+ if prop not in row:
53
+ row[prop] = None
54
+ return output
55
+
56
+
57
+ class DMSTableYamlExporter(DMSTableExporter, DMSFileExporter[DataModelTableType]):
58
+ """Exports DMS to YAML."""
59
+
60
+ def __init__(self) -> None:
61
+ super().__init__(exclude_none=True)
62
+
63
+ def export_to_file(self, data_model: RequestSchema, file_path: Path) -> None:
64
+ """Exports the data model as a flat YAML file, which is identical to the spreadsheet representation
65
+
66
+ Args:
67
+ data_model (RequestSchema): The data model to export.
68
+ file_path (Path): The path to the YAML file to create.
69
+ """
70
+ table_format = self.export(data_model)
71
+ file_path.write_text(
72
+ yaml.safe_dump(table_format, sort_keys=False), encoding=self.ENCODING, newline=self.NEW_LINE
73
+ )
74
+
75
+
76
+ class DMSTableJSONExporter(DMSTableExporter, DMSFileExporter[DataModelTableType]):
77
+ """Exports DMS to JSON."""
78
+
79
+ def __init__(self) -> None:
80
+ super().__init__(exclude_none=True)
81
+
82
+ def export_to_file(self, data_model: RequestSchema, file_path: Path) -> None:
83
+ """Exports the data model as a flat JSON file, which is identical to the spreadsheet representation
84
+
85
+ Args:
86
+ data_model (RequestSchema): The data model to export.
87
+ file_path (Path): The path to the JSON file to create.
88
+ """
89
+ table_format = self.export(data_model)
90
+ file_path.write_text(json.dumps(table_format), encoding=self.ENCODING, newline=self.NEW_LINE)
91
+
92
+
93
+ class DMSExcelExporter(DMSTableExporter, DMSFileExporter[DataModelTableType]):
94
+ """Exports DMS to Excel file."""
95
+
96
+ def __init__(self, options: WorkbookOptions | None = None) -> None:
97
+ self._options = options or WorkbookOptions()
98
+ super().__init__(
99
+ exclude_none=False, skip_properties_in_other_spaces=self._options.skip_properties_in_other_spaces
100
+ )
101
+
102
+ def export_to_file(self, data_model: RequestSchema, file_path: Path) -> None:
103
+ """Exports the data model as a Excel file.
104
+
105
+ Args:
106
+ data_model (RequestSchema): The data model to export.
107
+ file_path (Path): The path to the Excel file to create.
108
+ options (WorkbookOptions | None): Options for creating the workbook.
109
+ """
110
+ table_format = self.export(data_model)
111
+ workbook = WorkbookCreator(self._options).create_workbook(table_format)
112
+ try:
113
+ workbook.save(file_path)
114
+ finally:
115
+ workbook.close()
116
+
117
+
118
+ class DMSCsvExporter(DMSTableExporter, DMSFileExporter[DataModelTableType]):
119
+ """Exports DMS to CSV files in a directory."""
120
+
121
+ def export_to_file(self, data_model: RequestSchema, directory_path: Path) -> None:
122
+ """Exports the data model as a set of CSV files, one for each table.
123
+
124
+ Args:
125
+ data_model (RequestSchema): The data model to export.
126
+ directory_path (Path): The path to the directory to create the CSV files in.
127
+ """
128
+ raise NotImplementedError()