cognite-neat 0.123.2__py3-none-any.whl → 0.127.30__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (333) hide show
  1. cognite/neat/__init__.py +2 -2
  2. cognite/neat/_client/__init__.py +4 -0
  3. cognite/neat/_client/api.py +8 -0
  4. cognite/neat/_client/client.py +21 -0
  5. cognite/neat/_client/config.py +40 -0
  6. cognite/neat/_client/containers_api.py +125 -0
  7. cognite/neat/_client/data_classes.py +44 -0
  8. cognite/neat/_client/data_model_api.py +115 -0
  9. cognite/neat/_client/spaces_api.py +115 -0
  10. cognite/neat/_client/statistics_api.py +24 -0
  11. cognite/neat/_client/views_api.py +129 -0
  12. cognite/neat/_config.py +185 -0
  13. cognite/neat/_data_model/_analysis.py +196 -0
  14. cognite/neat/_data_model/_constants.py +67 -0
  15. cognite/neat/_data_model/_identifiers.py +61 -0
  16. cognite/neat/_data_model/_shared.py +41 -0
  17. cognite/neat/_data_model/deployer/_differ.py +140 -0
  18. cognite/neat/_data_model/deployer/_differ_container.py +360 -0
  19. cognite/neat/_data_model/deployer/_differ_data_model.py +54 -0
  20. cognite/neat/_data_model/deployer/_differ_space.py +9 -0
  21. cognite/neat/_data_model/deployer/_differ_view.py +299 -0
  22. cognite/neat/_data_model/deployer/data_classes.py +529 -0
  23. cognite/neat/_data_model/deployer/deployer.py +401 -0
  24. cognite/neat/_data_model/exporters/__init__.py +15 -0
  25. cognite/neat/_data_model/exporters/_api_exporter.py +37 -0
  26. cognite/neat/_data_model/exporters/_base.py +24 -0
  27. cognite/neat/_data_model/exporters/_table_exporter/exporter.py +128 -0
  28. cognite/neat/_data_model/exporters/_table_exporter/workbook.py +409 -0
  29. cognite/neat/_data_model/exporters/_table_exporter/writer.py +421 -0
  30. cognite/neat/_data_model/importers/__init__.py +5 -0
  31. cognite/neat/_data_model/importers/_api_importer.py +166 -0
  32. cognite/neat/_data_model/importers/_base.py +16 -0
  33. cognite/neat/_data_model/importers/_table_importer/data_classes.py +295 -0
  34. cognite/neat/_data_model/importers/_table_importer/importer.py +192 -0
  35. cognite/neat/_data_model/importers/_table_importer/reader.py +1063 -0
  36. cognite/neat/_data_model/importers/_table_importer/source.py +94 -0
  37. cognite/neat/_data_model/models/conceptual/_base.py +18 -0
  38. cognite/neat/_data_model/models/conceptual/_concept.py +67 -0
  39. cognite/neat/_data_model/models/conceptual/_data_model.py +51 -0
  40. cognite/neat/_data_model/models/conceptual/_properties.py +104 -0
  41. cognite/neat/_data_model/models/conceptual/_property.py +105 -0
  42. cognite/neat/_data_model/models/dms/__init__.py +206 -0
  43. cognite/neat/_data_model/models/dms/_base.py +31 -0
  44. cognite/neat/_data_model/models/dms/_constants.py +48 -0
  45. cognite/neat/_data_model/models/dms/_constraints.py +42 -0
  46. cognite/neat/_data_model/models/dms/_container.py +159 -0
  47. cognite/neat/_data_model/models/dms/_data_model.py +95 -0
  48. cognite/neat/_data_model/models/dms/_data_types.py +195 -0
  49. cognite/neat/_data_model/models/dms/_http.py +28 -0
  50. cognite/neat/_data_model/models/dms/_indexes.py +30 -0
  51. cognite/neat/_data_model/models/dms/_limits.py +96 -0
  52. cognite/neat/_data_model/models/dms/_references.py +135 -0
  53. cognite/neat/_data_model/models/dms/_schema.py +18 -0
  54. cognite/neat/_data_model/models/dms/_space.py +48 -0
  55. cognite/neat/_data_model/models/dms/_types.py +17 -0
  56. cognite/neat/_data_model/models/dms/_view_filter.py +282 -0
  57. cognite/neat/_data_model/models/dms/_view_property.py +235 -0
  58. cognite/neat/_data_model/models/dms/_views.py +210 -0
  59. cognite/neat/_data_model/models/entities/__init__.py +50 -0
  60. cognite/neat/_data_model/models/entities/_base.py +101 -0
  61. cognite/neat/_data_model/models/entities/_constants.py +22 -0
  62. cognite/neat/_data_model/models/entities/_data_types.py +144 -0
  63. cognite/neat/_data_model/models/entities/_identifiers.py +61 -0
  64. cognite/neat/_data_model/models/entities/_parser.py +226 -0
  65. cognite/neat/_data_model/validation/dms/__init__.py +75 -0
  66. cognite/neat/_data_model/validation/dms/_ai_readiness.py +364 -0
  67. cognite/neat/_data_model/validation/dms/_base.py +307 -0
  68. cognite/neat/_data_model/validation/dms/_connections.py +638 -0
  69. cognite/neat/_data_model/validation/dms/_consistency.py +57 -0
  70. cognite/neat/_data_model/validation/dms/_containers.py +174 -0
  71. cognite/neat/_data_model/validation/dms/_limits.py +420 -0
  72. cognite/neat/_data_model/validation/dms/_orchestrator.py +222 -0
  73. cognite/neat/_data_model/validation/dms/_views.py +103 -0
  74. cognite/neat/_exceptions.py +56 -0
  75. cognite/neat/_issues.py +68 -0
  76. cognite/neat/_session/__init__.py +3 -0
  77. cognite/neat/_session/_html/_render.py +30 -0
  78. cognite/neat/_session/_html/static/__init__.py +8 -0
  79. cognite/neat/_session/_html/static/deployment.css +303 -0
  80. cognite/neat/_session/_html/static/deployment.js +150 -0
  81. cognite/neat/_session/_html/static/issues.css +211 -0
  82. cognite/neat/_session/_html/static/issues.js +168 -0
  83. cognite/neat/_session/_html/static/shared.css +186 -0
  84. cognite/neat/_session/_html/templates/__init__.py +4 -0
  85. cognite/neat/_session/_html/templates/deployment.html +75 -0
  86. cognite/neat/_session/_html/templates/issues.html +45 -0
  87. cognite/neat/_session/_issues.py +81 -0
  88. cognite/neat/_session/_opt.py +35 -0
  89. cognite/neat/_session/_physical.py +261 -0
  90. cognite/neat/_session/_result.py +236 -0
  91. cognite/neat/_session/_session.py +88 -0
  92. cognite/neat/_session/_usage_analytics/__init__.py +0 -0
  93. cognite/neat/_session/_usage_analytics/_collector.py +131 -0
  94. cognite/neat/_session/_usage_analytics/_constants.py +23 -0
  95. cognite/neat/_session/_usage_analytics/_storage.py +240 -0
  96. cognite/neat/_session/_wrappers.py +82 -0
  97. cognite/neat/_state_machine/__init__.py +10 -0
  98. cognite/neat/_state_machine/_base.py +37 -0
  99. cognite/neat/_state_machine/_states.py +52 -0
  100. cognite/neat/_store/__init__.py +3 -0
  101. cognite/neat/_store/_provenance.py +81 -0
  102. cognite/neat/_store/_store.py +156 -0
  103. cognite/neat/_utils/__init__.py +0 -0
  104. cognite/neat/_utils/_reader.py +194 -0
  105. cognite/neat/_utils/auxiliary.py +39 -0
  106. cognite/neat/_utils/collection.py +11 -0
  107. cognite/neat/_utils/http_client/__init__.py +39 -0
  108. cognite/neat/_utils/http_client/_client.py +245 -0
  109. cognite/neat/_utils/http_client/_config.py +19 -0
  110. cognite/neat/_utils/http_client/_data_classes.py +294 -0
  111. cognite/neat/_utils/http_client/_tracker.py +31 -0
  112. cognite/neat/_utils/text.py +71 -0
  113. cognite/neat/_utils/useful_types.py +37 -0
  114. cognite/neat/_utils/validation.py +154 -0
  115. cognite/neat/_version.py +1 -1
  116. cognite/neat/v0/__init__.py +0 -0
  117. cognite/neat/v0/core/__init__.py +0 -0
  118. cognite/neat/v0/core/_client/_api/__init__.py +0 -0
  119. cognite/neat/{core → v0/core}/_client/_api/data_modeling_loaders.py +86 -7
  120. cognite/neat/{core → v0/core}/_client/_api/neat_instances.py +5 -5
  121. cognite/neat/{core → v0/core}/_client/_api/schema.py +5 -5
  122. cognite/neat/{core → v0/core}/_client/_api/statistics.py +3 -3
  123. cognite/neat/{core → v0/core}/_client/_api_client.py +1 -1
  124. cognite/neat/v0/core/_client/data_classes/__init__.py +0 -0
  125. cognite/neat/{core → v0/core}/_client/data_classes/schema.py +4 -4
  126. cognite/neat/{core → v0/core}/_client/testing.py +1 -1
  127. cognite/neat/{core → v0/core}/_constants.py +10 -3
  128. cognite/neat/v0/core/_data_model/__init__.py +0 -0
  129. cognite/neat/{core → v0/core}/_data_model/_constants.py +9 -6
  130. cognite/neat/{core → v0/core}/_data_model/_shared.py +5 -5
  131. cognite/neat/{core → v0/core}/_data_model/analysis/_base.py +12 -8
  132. cognite/neat/{core → v0/core}/_data_model/exporters/__init__.py +1 -2
  133. cognite/neat/{core → v0/core}/_data_model/exporters/_base.py +7 -7
  134. cognite/neat/{core → v0/core}/_data_model/exporters/_data_model2dms.py +9 -9
  135. cognite/neat/{core → v0/core}/_data_model/exporters/_data_model2excel.py +13 -13
  136. cognite/neat/{core → v0/core}/_data_model/exporters/_data_model2instance_template.py +4 -4
  137. cognite/neat/{core/_data_model/exporters/_data_model2ontology.py → v0/core/_data_model/exporters/_data_model2semantic_model.py} +126 -133
  138. cognite/neat/{core → v0/core}/_data_model/exporters/_data_model2yaml.py +1 -1
  139. cognite/neat/{core → v0/core}/_data_model/importers/__init__.py +4 -6
  140. cognite/neat/{core → v0/core}/_data_model/importers/_base.py +5 -5
  141. cognite/neat/{core → v0/core}/_data_model/importers/_base_file_reader.py +2 -2
  142. cognite/neat/{core → v0/core}/_data_model/importers/_dict2data_model.py +6 -6
  143. cognite/neat/{core → v0/core}/_data_model/importers/_dms2data_model.py +19 -16
  144. cognite/neat/v0/core/_data_model/importers/_graph2data_model.py +299 -0
  145. cognite/neat/v0/core/_data_model/importers/_rdf/__init__.py +4 -0
  146. cognite/neat/{core → v0/core}/_data_model/importers/_rdf/_base.py +13 -13
  147. cognite/neat/{core → v0/core}/_data_model/importers/_rdf/_inference2rdata_model.py +14 -14
  148. cognite/neat/v0/core/_data_model/importers/_rdf/_owl2data_model.py +144 -0
  149. cognite/neat/v0/core/_data_model/importers/_rdf/_shared.py +255 -0
  150. cognite/neat/{core → v0/core}/_data_model/importers/_spreadsheet2data_model.py +94 -13
  151. cognite/neat/{core → v0/core}/_data_model/models/__init__.py +3 -3
  152. cognite/neat/{core → v0/core}/_data_model/models/_base_verified.py +5 -5
  153. cognite/neat/v0/core/_data_model/models/_import_contexts.py +82 -0
  154. cognite/neat/{core → v0/core}/_data_model/models/_types.py +5 -5
  155. cognite/neat/{core → v0/core}/_data_model/models/conceptual/_unverified.py +18 -12
  156. cognite/neat/v0/core/_data_model/models/conceptual/_validation.py +308 -0
  157. cognite/neat/{core → v0/core}/_data_model/models/conceptual/_verified.py +13 -11
  158. cognite/neat/{core → v0/core}/_data_model/models/data_types.py +14 -4
  159. cognite/neat/{core → v0/core}/_data_model/models/entities/__init__.py +6 -0
  160. cognite/neat/v0/core/_data_model/models/entities/_loaders.py +155 -0
  161. cognite/neat/{core → v0/core}/_data_model/models/entities/_multi_value.py +2 -2
  162. cognite/neat/v0/core/_data_model/models/entities/_restrictions.py +230 -0
  163. cognite/neat/{core → v0/core}/_data_model/models/entities/_single_value.py +121 -16
  164. cognite/neat/{core → v0/core}/_data_model/models/entities/_types.py +10 -0
  165. cognite/neat/{core → v0/core}/_data_model/models/mapping/_classic2core.py +5 -5
  166. cognite/neat/{core → v0/core}/_data_model/models/physical/__init__.py +1 -1
  167. cognite/neat/{core → v0/core}/_data_model/models/physical/_exporter.py +28 -21
  168. cognite/neat/{core → v0/core}/_data_model/models/physical/_unverified.py +141 -38
  169. cognite/neat/{core → v0/core}/_data_model/models/physical/_validation.py +190 -24
  170. cognite/neat/{core → v0/core}/_data_model/models/physical/_verified.py +135 -15
  171. cognite/neat/{core → v0/core}/_data_model/transformers/__init__.py +2 -0
  172. cognite/neat/{core → v0/core}/_data_model/transformers/_base.py +4 -4
  173. cognite/neat/{core → v0/core}/_data_model/transformers/_converters.py +39 -32
  174. cognite/neat/{core → v0/core}/_data_model/transformers/_mapping.py +7 -7
  175. cognite/neat/v0/core/_data_model/transformers/_union_conceptual.py +208 -0
  176. cognite/neat/{core → v0/core}/_data_model/transformers/_verification.py +7 -7
  177. cognite/neat/v0/core/_instances/__init__.py +0 -0
  178. cognite/neat/{core → v0/core}/_instances/_tracking/base.py +1 -1
  179. cognite/neat/{core → v0/core}/_instances/_tracking/log.py +1 -1
  180. cognite/neat/{core → v0/core}/_instances/extractors/__init__.py +1 -1
  181. cognite/neat/{core → v0/core}/_instances/extractors/_base.py +6 -6
  182. cognite/neat/v0/core/_instances/extractors/_classic_cdf/__init__.py +0 -0
  183. cognite/neat/{core → v0/core}/_instances/extractors/_classic_cdf/_base.py +7 -7
  184. cognite/neat/{core → v0/core}/_instances/extractors/_classic_cdf/_classic.py +12 -12
  185. cognite/neat/{core → v0/core}/_instances/extractors/_classic_cdf/_relationships.py +3 -3
  186. cognite/neat/{core → v0/core}/_instances/extractors/_classic_cdf/_sequences.py +2 -2
  187. cognite/neat/{core → v0/core}/_instances/extractors/_dict.py +6 -3
  188. cognite/neat/{core → v0/core}/_instances/extractors/_dms.py +6 -6
  189. cognite/neat/{core → v0/core}/_instances/extractors/_dms_graph.py +11 -11
  190. cognite/neat/{core → v0/core}/_instances/extractors/_mock_graph_generator.py +10 -10
  191. cognite/neat/{core → v0/core}/_instances/extractors/_raw.py +3 -3
  192. cognite/neat/{core → v0/core}/_instances/extractors/_rdf_file.py +7 -7
  193. cognite/neat/{core → v0/core}/_instances/loaders/_base.py +5 -5
  194. cognite/neat/{core → v0/core}/_instances/loaders/_rdf2dms.py +17 -17
  195. cognite/neat/{core → v0/core}/_instances/loaders/_rdf_to_instance_space.py +11 -11
  196. cognite/neat/{core → v0/core}/_instances/queries/_select.py +29 -3
  197. cognite/neat/{core → v0/core}/_instances/queries/_update.py +1 -1
  198. cognite/neat/{core → v0/core}/_instances/transformers/_base.py +4 -4
  199. cognite/neat/{core → v0/core}/_instances/transformers/_classic_cdf.py +6 -6
  200. cognite/neat/{core → v0/core}/_instances/transformers/_prune_graph.py +4 -4
  201. cognite/neat/{core → v0/core}/_instances/transformers/_rdfpath.py +1 -1
  202. cognite/neat/{core → v0/core}/_instances/transformers/_value_type.py +4 -4
  203. cognite/neat/{core → v0/core}/_issues/_base.py +11 -6
  204. cognite/neat/{core → v0/core}/_issues/_contextmanagers.py +8 -6
  205. cognite/neat/{core → v0/core}/_issues/_factory.py +11 -8
  206. cognite/neat/{core → v0/core}/_issues/errors/__init__.py +3 -1
  207. cognite/neat/{core → v0/core}/_issues/errors/_external.py +1 -1
  208. cognite/neat/{core → v0/core}/_issues/errors/_general.py +1 -1
  209. cognite/neat/{core → v0/core}/_issues/errors/_properties.py +12 -1
  210. cognite/neat/{core → v0/core}/_issues/errors/_resources.py +2 -2
  211. cognite/neat/{core → v0/core}/_issues/errors/_wrapper.py +7 -3
  212. cognite/neat/{core → v0/core}/_issues/warnings/__init__.py +5 -1
  213. cognite/neat/{core → v0/core}/_issues/warnings/_external.py +1 -1
  214. cognite/neat/{core → v0/core}/_issues/warnings/_general.py +1 -1
  215. cognite/neat/{core → v0/core}/_issues/warnings/_models.py +39 -4
  216. cognite/neat/{core → v0/core}/_issues/warnings/_properties.py +13 -2
  217. cognite/neat/{core → v0/core}/_issues/warnings/_resources.py +1 -1
  218. cognite/neat/{core → v0/core}/_issues/warnings/user_modeling.py +1 -1
  219. cognite/neat/{core → v0/core}/_store/_data_model.py +13 -12
  220. cognite/neat/{core → v0/core}/_store/_instance.py +45 -12
  221. cognite/neat/{core → v0/core}/_store/_provenance.py +3 -3
  222. cognite/neat/{core → v0/core}/_store/exceptions.py +4 -4
  223. cognite/neat/v0/core/_utils/__init__.py +0 -0
  224. cognite/neat/{core → v0/core}/_utils/auth.py +1 -1
  225. cognite/neat/{core → v0/core}/_utils/auxiliary.py +7 -1
  226. cognite/neat/{core → v0/core}/_utils/collection_.py +2 -2
  227. cognite/neat/{core → v0/core}/_utils/graph_transformations_report.py +1 -1
  228. cognite/neat/{core → v0/core}/_utils/rdf_.py +38 -14
  229. cognite/neat/{core → v0/core}/_utils/reader/_base.py +1 -1
  230. cognite/neat/{core → v0/core}/_utils/spreadsheet.py +22 -4
  231. cognite/neat/v0/core/_utils/tarjan.py +44 -0
  232. cognite/neat/{core → v0/core}/_utils/text.py +1 -1
  233. cognite/neat/{core → v0/core}/_utils/upload.py +3 -3
  234. cognite/neat/v0/plugins/__init__.py +4 -0
  235. cognite/neat/v0/plugins/_base.py +9 -0
  236. cognite/neat/v0/plugins/_data_model.py +48 -0
  237. cognite/neat/{plugins → v0/plugins}/_issues.py +1 -1
  238. cognite/neat/{plugins → v0/plugins}/_manager.py +7 -16
  239. cognite/neat/{session → v0/session}/_base.py +13 -10
  240. cognite/neat/{session → v0/session}/_collector.py +1 -1
  241. cognite/neat/v0/session/_diff.py +51 -0
  242. cognite/neat/{session → v0/session}/_drop.py +3 -3
  243. cognite/neat/{session → v0/session}/_explore.py +2 -2
  244. cognite/neat/{session → v0/session}/_fix.py +2 -2
  245. cognite/neat/{session → v0/session}/_inspect.py +3 -3
  246. cognite/neat/{session → v0/session}/_mapping.py +3 -3
  247. cognite/neat/{session → v0/session}/_plugin.py +4 -5
  248. cognite/neat/{session → v0/session}/_prepare.py +8 -8
  249. cognite/neat/{session → v0/session}/_read.py +33 -43
  250. cognite/neat/{session → v0/session}/_set.py +8 -8
  251. cognite/neat/{session → v0/session}/_show.py +5 -5
  252. cognite/neat/{session → v0/session}/_state.py +22 -8
  253. cognite/neat/{session → v0/session}/_subset.py +4 -4
  254. cognite/neat/{session → v0/session}/_template.py +11 -11
  255. cognite/neat/{session → v0/session}/_to.py +12 -12
  256. cognite/neat/{session → v0/session}/_wizard.py +1 -1
  257. cognite/neat/{session → v0/session}/engine/_load.py +1 -1
  258. cognite/neat/{session → v0/session}/exceptions.py +5 -5
  259. cognite/neat/v1.py +3 -0
  260. {cognite_neat-0.123.2.dist-info → cognite_neat-0.127.30.dist-info}/METADATA +9 -8
  261. cognite_neat-0.127.30.dist-info/RECORD +319 -0
  262. {cognite_neat-0.123.2.dist-info → cognite_neat-0.127.30.dist-info}/WHEEL +1 -1
  263. cognite/neat/core/_data_model/importers/_dtdl2data_model/__init__.py +0 -3
  264. cognite/neat/core/_data_model/importers/_dtdl2data_model/_unit_lookup.py +0 -224
  265. cognite/neat/core/_data_model/importers/_dtdl2data_model/dtdl_converter.py +0 -320
  266. cognite/neat/core/_data_model/importers/_dtdl2data_model/dtdl_importer.py +0 -155
  267. cognite/neat/core/_data_model/importers/_dtdl2data_model/spec.py +0 -363
  268. cognite/neat/core/_data_model/importers/_rdf/__init__.py +0 -5
  269. cognite/neat/core/_data_model/importers/_rdf/_imf2data_model.py +0 -98
  270. cognite/neat/core/_data_model/importers/_rdf/_owl2data_model.py +0 -87
  271. cognite/neat/core/_data_model/importers/_rdf/_shared.py +0 -168
  272. cognite/neat/core/_data_model/models/conceptual/_validation.py +0 -294
  273. cognite/neat/core/_data_model/models/entities/_loaders.py +0 -75
  274. cognite/neat/plugins/__init__.py +0 -3
  275. cognite/neat/plugins/data_model/importers/__init__.py +0 -5
  276. cognite/neat/plugins/data_model/importers/_base.py +0 -28
  277. cognite_neat-0.123.2.dist-info/RECORD +0 -197
  278. /cognite/neat/{core → _data_model}/__init__.py +0 -0
  279. /cognite/neat/{core/_client/_api → _data_model/deployer}/__init__.py +0 -0
  280. /cognite/neat/{core/_client/data_classes → _data_model/exporters/_table_exporter}/__init__.py +0 -0
  281. /cognite/neat/{core/_data_model → _data_model/importers/_table_importer}/__init__.py +0 -0
  282. /cognite/neat/{core/_instances → _data_model/models}/__init__.py +0 -0
  283. /cognite/neat/{core/_instances/extractors/_classic_cdf → _data_model/models/conceptual}/__init__.py +0 -0
  284. /cognite/neat/{core/_utils → _data_model/validation}/__init__.py +0 -0
  285. /cognite/neat/{plugins/data_model → _session/_html}/__init__.py +0 -0
  286. /cognite/neat/{core → v0/core}/_client/__init__.py +0 -0
  287. /cognite/neat/{core → v0/core}/_client/data_classes/data_modeling.py +0 -0
  288. /cognite/neat/{core → v0/core}/_client/data_classes/neat_sequence.py +0 -0
  289. /cognite/neat/{core → v0/core}/_client/data_classes/statistics.py +0 -0
  290. /cognite/neat/{core → v0/core}/_config.py +0 -0
  291. /cognite/neat/{core → v0/core}/_data_model/analysis/__init__.py +0 -0
  292. /cognite/neat/{core → v0/core}/_data_model/catalog/__init__.py +0 -0
  293. /cognite/neat/{core → v0/core}/_data_model/catalog/classic_model.xlsx +0 -0
  294. /cognite/neat/{core → v0/core}/_data_model/catalog/conceptual-imf-data-model.xlsx +0 -0
  295. /cognite/neat/{core → v0/core}/_data_model/catalog/hello_world_pump.xlsx +0 -0
  296. /cognite/neat/{core → v0/core}/_data_model/models/_base_unverified.py +0 -0
  297. /cognite/neat/{core → v0/core}/_data_model/models/conceptual/__init__.py +0 -0
  298. /cognite/neat/{core → v0/core}/_data_model/models/entities/_constants.py +0 -0
  299. /cognite/neat/{core → v0/core}/_data_model/models/entities/_wrapped.py +0 -0
  300. /cognite/neat/{core → v0/core}/_data_model/models/mapping/__init__.py +0 -0
  301. /cognite/neat/{core → v0/core}/_data_model/models/mapping/_classic2core.yaml +0 -0
  302. /cognite/neat/{core → v0/core}/_instances/_shared.py +0 -0
  303. /cognite/neat/{core → v0/core}/_instances/_tracking/__init__.py +0 -0
  304. /cognite/neat/{core → v0/core}/_instances/examples/Knowledge-Graph-Nordic44-dirty.xml +0 -0
  305. /cognite/neat/{core → v0/core}/_instances/examples/Knowledge-Graph-Nordic44.xml +0 -0
  306. /cognite/neat/{core → v0/core}/_instances/examples/__init__.py +0 -0
  307. /cognite/neat/{core → v0/core}/_instances/examples/skos-capturing-sheet-wind-topics.xlsx +0 -0
  308. /cognite/neat/{core → v0/core}/_instances/extractors/_classic_cdf/_assets.py +0 -0
  309. /cognite/neat/{core → v0/core}/_instances/extractors/_classic_cdf/_data_sets.py +0 -0
  310. /cognite/neat/{core → v0/core}/_instances/extractors/_classic_cdf/_events.py +0 -0
  311. /cognite/neat/{core → v0/core}/_instances/extractors/_classic_cdf/_files.py +0 -0
  312. /cognite/neat/{core → v0/core}/_instances/extractors/_classic_cdf/_labels.py +0 -0
  313. /cognite/neat/{core → v0/core}/_instances/extractors/_classic_cdf/_timeseries.py +0 -0
  314. /cognite/neat/{core → v0/core}/_instances/loaders/__init__.py +0 -0
  315. /cognite/neat/{core → v0/core}/_instances/queries/__init__.py +0 -0
  316. /cognite/neat/{core → v0/core}/_instances/queries/_base.py +0 -0
  317. /cognite/neat/{core → v0/core}/_instances/queries/_queries.py +0 -0
  318. /cognite/neat/{core → v0/core}/_instances/transformers/__init__.py +0 -0
  319. /cognite/neat/{core → v0/core}/_issues/__init__.py +0 -0
  320. /cognite/neat/{core → v0/core}/_issues/formatters.py +0 -0
  321. /cognite/neat/{core → v0/core}/_shared.py +0 -0
  322. /cognite/neat/{core → v0/core}/_store/__init__.py +0 -0
  323. /cognite/neat/{core → v0/core}/_utils/io_.py +0 -0
  324. /cognite/neat/{core → v0/core}/_utils/reader/__init__.py +0 -0
  325. /cognite/neat/{core → v0/core}/_utils/time_.py +0 -0
  326. /cognite/neat/{core → v0/core}/_utils/xml_.py +0 -0
  327. /cognite/neat/{session → v0/session}/__init__.py +0 -0
  328. /cognite/neat/{session → v0/session}/_experimental.py +0 -0
  329. /cognite/neat/{session → v0/session}/_state/README.md +0 -0
  330. /cognite/neat/{session → v0/session}/engine/__init__.py +0 -0
  331. /cognite/neat/{session → v0/session}/engine/_import.py +0 -0
  332. /cognite/neat/{session → v0/session}/engine/_interface.py +0 -0
  333. {cognite_neat-0.123.2.dist-info → cognite_neat-0.127.30.dist-info}/licenses/LICENSE +0 -0
@@ -3,23 +3,25 @@ import sys
3
3
  import warnings
4
4
  from dataclasses import dataclass
5
5
  from datetime import datetime
6
- from typing import Any, Literal
6
+ from typing import Any, Literal, overload
7
7
 
8
8
  import pandas as pd
9
9
  from cognite.client import data_modeling as dm
10
10
  from cognite.client.data_classes.data_modeling import ContainerId, ViewId
11
11
  from rdflib import Namespace, URIRef
12
12
 
13
- from cognite.neat.core._constants import (
13
+ from cognite.neat.v0.core._constants import (
14
14
  DEFAULT_NAMESPACE,
15
15
  DMS_DIRECT_RELATION_LIST_DEFAULT_LIMIT,
16
16
  )
17
- from cognite.neat.core._data_model.models._base_unverified import (
17
+ from cognite.neat.v0.core._data_model._constants import SPLIT_ON_COMMA_PATTERN
18
+ from cognite.neat.v0.core._data_model.models._base_unverified import (
18
19
  UnverifiedComponent,
19
20
  UnverifiedDataModel,
20
21
  )
21
- from cognite.neat.core._data_model.models.data_types import DataType
22
- from cognite.neat.core._data_model.models.entities import (
22
+ from cognite.neat.v0.core._data_model.models.data_types import DataType
23
+ from cognite.neat.v0.core._data_model.models.entities import (
24
+ ContainerConstraintEntity,
23
25
  ContainerEntity,
24
26
  ContainerIndexEntity,
25
27
  DMSNodeEntity,
@@ -30,9 +32,9 @@ from cognite.neat.core._data_model.models.entities import (
30
32
  load_connection,
31
33
  load_dms_value_type,
32
34
  )
33
- from cognite.neat.core._data_model.models.entities._wrapped import DMSFilter
34
- from cognite.neat.core._issues.warnings import DeprecatedWarning
35
- from cognite.neat.core._utils.rdf_ import uri_display_name
35
+ from cognite.neat.v0.core._data_model.models.entities._wrapped import DMSFilter
36
+ from cognite.neat.v0.core._issues.warnings import DeprecatedWarning
37
+ from cognite.neat.v0.core._utils.rdf_ import uri_display_name
36
38
 
37
39
  from ._verified import (
38
40
  _DEFAULT_VERSION,
@@ -137,7 +139,7 @@ class UnverifiedPhysicalProperty(UnverifiedComponent[PhysicalProperty]):
137
139
  container: str | None = None
138
140
  container_property: str | None = None
139
141
  index: str | list[str | ContainerIndexEntity] | ContainerIndexEntity | None = None
140
- constraint: str | list[str] | None = None
142
+ constraint: str | list[str] | list[ContainerConstraintEntity] | ContainerConstraintEntity | None = None
141
143
  neatId: str | URIRef | None = None
142
144
  conceptual: str | URIRef | None = None
143
145
 
@@ -160,34 +162,44 @@ class UnverifiedPhysicalProperty(UnverifiedComponent[PhysicalProperty]):
160
162
 
161
163
  def dump(self, default_space: str, default_version: str) -> dict[str, Any]: # type: ignore[override]
162
164
  output = super().dump()
163
- output["View"] = ViewEntity.load(self.view, space=default_space, version=default_version)
164
- output["Value Type"] = load_dms_value_type(self.value_type, default_space, default_version)
165
- output["Connection"] = load_connection(self.connection, default_space, default_version)
165
+ output["View"] = ViewEntity.load(
166
+ self.view, space=default_space, version=default_version, return_on_failure=True
167
+ )
168
+ output["Value Type"] = load_dms_value_type(
169
+ self.value_type, default_space, default_version, return_on_failure=True
170
+ )
171
+ output["Connection"] = load_connection(self.connection, default_space, default_version, return_on_failure=True)
166
172
  output["Container"] = (
167
- ContainerEntity.load(self.container, space=default_space, version=default_version)
173
+ ContainerEntity.load(self.container, space=default_space, version=default_version, return_on_failure=True)
168
174
  if self.container
169
175
  else None
170
176
  )
171
177
  if isinstance(self.index, ContainerIndexEntity) or (isinstance(self.index, str) and "," not in self.index):
172
- output["Index"] = [ContainerIndexEntity.load(self.index)]
173
- elif isinstance(self.index, str) and "," in self.index:
178
+ output["Index"] = [ContainerIndexEntity.load(self.index, return_on_failure=True)]
179
+ elif isinstance(self.index, str):
174
180
  output["Index"] = [
175
- ContainerIndexEntity.load(index.strip()) for index in self.index.split(",") if index.strip()
181
+ ContainerIndexEntity.load(index.strip(), return_on_failure=True)
182
+ for index in SPLIT_ON_COMMA_PATTERN.split(self.index)
183
+ if index.strip()
176
184
  ]
177
185
  elif isinstance(self.index, list):
178
186
  index_list: list[ContainerIndexEntity | PhysicalUnknownEntity] = []
179
187
  for index in self.index:
180
188
  if isinstance(index, ContainerIndexEntity):
181
189
  index_list.append(index)
182
- elif isinstance(index, str) and "," in index:
190
+ elif isinstance(index, str):
183
191
  index_list.extend(
184
- [ContainerIndexEntity.load(idx.strip()) for idx in index.split(",") if idx.strip()]
192
+ [
193
+ ContainerIndexEntity.load(idx.strip(), return_on_failure=True)
194
+ for idx in SPLIT_ON_COMMA_PATTERN.split(index)
195
+ if idx.strip()
196
+ ]
185
197
  )
186
- elif isinstance(index, str):
187
- index_list.append(ContainerIndexEntity.load(index.strip()))
188
198
  else:
189
199
  raise TypeError(f"Unexpected type for index: {type(index)}")
190
200
  output["Index"] = index_list
201
+
202
+ output["Constraint"] = _parse_constraints(self.constraint, default_space)
191
203
  return output
192
204
 
193
205
  def referenced_view(self, default_space: str, default_version: str) -> ViewEntity:
@@ -240,7 +252,7 @@ class UnverifiedPhysicalContainer(UnverifiedComponent[PhysicalContainer]):
240
252
  container: str
241
253
  name: str | None = None
242
254
  description: str | None = None
243
- constraint: str | None = None
255
+ constraint: str | list[str] | list[ContainerConstraintEntity] | ContainerConstraintEntity | None = None
244
256
  neatId: str | URIRef | None = None
245
257
  used_for: Literal["node", "edge", "all"] | None = None
246
258
 
@@ -250,23 +262,33 @@ class UnverifiedPhysicalContainer(UnverifiedComponent[PhysicalContainer]):
250
262
 
251
263
  def dump(self, default_space: str) -> dict[str, Any]: # type: ignore[override]
252
264
  output = super().dump()
253
- output["Container"] = self.as_entity_id(default_space)
254
- output["Constraint"] = (
255
- [ContainerEntity.load(constraint.strip(), space=default_space) for constraint in self.constraint.split(",")]
256
- if self.constraint
257
- else None
258
- )
265
+ output["Container"] = self.as_entity_id(default_space, return_on_failure=True)
266
+ output["Constraint"] = _parse_constraints(self.constraint, default_space)
259
267
  return output
260
268
 
261
- def as_entity_id(self, default_space: str) -> ContainerEntity:
262
- return ContainerEntity.load(self.container, strict=True, space=default_space)
269
+ @overload
270
+ def as_entity_id(self, default_space: str, return_on_failure: Literal[False] = False) -> ContainerEntity: ...
271
+
272
+ @overload
273
+ def as_entity_id(self, default_space: str, return_on_failure: Literal[True]) -> ContainerEntity | str: ...
274
+
275
+ def as_entity_id(
276
+ self, default_space: str, return_on_failure: Literal[True, False] = False
277
+ ) -> ContainerEntity | str:
278
+ return ContainerEntity.load(
279
+ self.container, strict=True, space=default_space, return_on_failure=return_on_failure
280
+ )
263
281
 
264
282
  @classmethod
265
283
  def from_container(cls, container: dm.ContainerApply) -> "UnverifiedPhysicalContainer":
266
284
  constraints: list[str] = []
267
- for _, constraint_obj in (container.constraints or {}).items():
285
+ for constraint_name, constraint_obj in (container.constraints or {}).items():
268
286
  if isinstance(constraint_obj, dm.RequiresConstraint):
269
- constraints.append(str(ContainerEntity.from_id(constraint_obj.require)))
287
+ constraint = ContainerConstraintEntity(
288
+ prefix="requires", suffix=constraint_name, require=ContainerEntity.from_id(constraint_obj.require)
289
+ )
290
+ constraints.append(str(constraint))
291
+
270
292
  # UniquenessConstraint it handled in the properties
271
293
  container_entity = ContainerEntity.from_id(container.as_id())
272
294
  return cls(
@@ -299,19 +321,51 @@ class UnverifiedPhysicalView(UnverifiedComponent[PhysicalView]):
299
321
 
300
322
  def dump(self, default_space: str, default_version: str) -> dict[str, Any]: # type: ignore[override]
301
323
  output = super().dump()
302
- output["View"] = self.as_entity_id(default_space, default_version)
303
- output["Implements"] = self._load_implements(default_space, default_version)
324
+ output["View"] = self.as_entity_id(default_space, default_version, return_on_failure=True)
325
+ output["Implements"] = self._load_implements(default_space, default_version, return_on_failure=True)
304
326
  return output
305
327
 
306
- def as_entity_id(self, default_space: str, default_version: str) -> ViewEntity:
307
- return ViewEntity.load(self.view, strict=True, space=default_space, version=default_version)
328
+ @overload
329
+ def as_entity_id(
330
+ self, default_space: str, default_version: str, return_on_failure: Literal[False] = False
331
+ ) -> ViewEntity: ...
332
+
333
+ @overload
334
+ def as_entity_id(
335
+ self, default_space: str, default_version: str, return_on_failure: Literal[True]
336
+ ) -> ViewEntity | str: ...
337
+
338
+ def as_entity_id(
339
+ self, default_space: str, default_version: str, return_on_failure: Literal[True, False] = False
340
+ ) -> ViewEntity | str:
341
+ return ViewEntity.load(
342
+ self.view, strict=True, space=default_space, version=default_version, return_on_failure=return_on_failure
343
+ )
308
344
 
309
- def _load_implements(self, default_space: str, default_version: str) -> list[ViewEntity] | None:
345
+ @overload
346
+ def _load_implements(
347
+ self, default_space: str, default_version: str, return_on_failure: Literal[False] = False
348
+ ) -> list[ViewEntity] | None: ...
349
+
350
+ @overload
351
+ def _load_implements(
352
+ self, default_space: str, default_version: str, return_on_failure: Literal[True]
353
+ ) -> list[ViewEntity | str] | None: ...
354
+
355
+ def _load_implements(
356
+ self, default_space: str, default_version: str, return_on_failure: Literal[True, False] = False
357
+ ) -> list[ViewEntity] | list[ViewEntity | str] | None:
310
358
  self.implements = self.implements.strip() if self.implements else None
311
359
 
312
360
  return (
313
361
  [
314
- ViewEntity.load(implement, strict=True, space=default_space, version=default_version)
362
+ ViewEntity.load(
363
+ implement,
364
+ strict=True,
365
+ space=default_space,
366
+ version=default_version,
367
+ return_on_failure=return_on_failure,
368
+ )
315
369
  for implement in self.implements.split(",")
316
370
  ]
317
371
  if self.implements
@@ -354,7 +408,7 @@ class UnverifiedPhysicalNodeType(UnverifiedComponent[PhysicalNodeType]):
354
408
 
355
409
  def dump(self, default_space: str, **_) -> dict[str, Any]: # type: ignore
356
410
  output = super().dump()
357
- output["Node"] = DMSNodeEntity.load(self.node, space=default_space)
411
+ output["Node"] = DMSNodeEntity.load(self.node, space=default_space, return_on_failure=True)
358
412
  return output
359
413
 
360
414
 
@@ -452,3 +506,52 @@ class UnverifiedPhysicalDataModel(UnverifiedDataModel[PhysicalDataModel]):
452
506
  def imported_views_and_containers_ids(self) -> tuple[set[ViewId], set[ContainerId]]:
453
507
  views, containers = self.imported_views_and_containers()
454
508
  return {view.as_id() for view in views}, {container.as_id() for container in containers}
509
+
510
+
511
+ def _parse_constraints(
512
+ constraint: str | list[str] | list[ContainerConstraintEntity] | ContainerConstraintEntity | None,
513
+ default_space: str | None = None,
514
+ ) -> list[ContainerConstraintEntity | PhysicalUnknownEntity] | None:
515
+ """Parse constraint input into a standardized list of ContainerConstraintEntity objects.
516
+
517
+ Args:
518
+ constraint: The constraint input in various formats
519
+ default_space: Default space to use when loading constraint entities
520
+
521
+ Returns:
522
+ List of parsed constraint entities, or None if no constraints
523
+ """
524
+ if constraint is None:
525
+ return None
526
+
527
+ if isinstance(constraint, ContainerConstraintEntity):
528
+ return [constraint]
529
+
530
+ if isinstance(constraint, str) and "," not in constraint:
531
+ return [ContainerConstraintEntity.load(constraint, return_on_failure=True, space=default_space)]
532
+
533
+ if isinstance(constraint, str):
534
+ return [
535
+ ContainerConstraintEntity.load(constraint_item.strip(), return_on_failure=True, space=default_space)
536
+ for constraint_item in SPLIT_ON_COMMA_PATTERN.split(constraint)
537
+ if constraint_item.strip()
538
+ ]
539
+
540
+ if isinstance(constraint, list):
541
+ constraint_list: list[ContainerConstraintEntity | PhysicalUnknownEntity] = []
542
+ for constraint_item in constraint:
543
+ if isinstance(constraint_item, ContainerConstraintEntity):
544
+ constraint_list.append(constraint_item)
545
+ elif isinstance(constraint_item, str):
546
+ constraint_list.extend(
547
+ [
548
+ ContainerConstraintEntity.load(idx.strip(), return_on_failure=True, space=default_space)
549
+ for idx in SPLIT_ON_COMMA_PATTERN.split(constraint_item)
550
+ if idx.strip()
551
+ ]
552
+ )
553
+ else:
554
+ raise TypeError(f"Unexpected type for constraint: {type(constraint_item)}")
555
+ return constraint_list
556
+
557
+ raise TypeError(f"Unexpected type for constraint: {type(constraint)}")
@@ -1,7 +1,9 @@
1
1
  import warnings
2
2
  from collections import Counter, defaultdict
3
3
  from collections.abc import Mapping
4
+ from dataclasses import dataclass
4
5
  from functools import lru_cache
6
+ from typing import cast
5
7
 
6
8
  from cognite.client import data_modeling as dm
7
9
  from cognite.client.data_classes.data_modeling import ContainerList, ViewId, ViewList
@@ -12,49 +14,68 @@ from cognite.client.data_classes.data_modeling.views import (
12
14
  ViewPropertyApply,
13
15
  )
14
16
 
15
- from cognite.neat.core._client import NeatClient
16
- from cognite.neat.core._client.data_classes.data_modeling import ViewApplyDict
17
- from cognite.neat.core._client.data_classes.schema import DMSSchema
18
- from cognite.neat.core._constants import (
17
+ from cognite.neat.v0.core._client import NeatClient
18
+ from cognite.neat.v0.core._client.data_classes.data_modeling import ViewApplyDict
19
+ from cognite.neat.v0.core._client.data_classes.schema import DMSSchema
20
+ from cognite.neat.v0.core._constants import (
19
21
  COGNITE_MODELS,
20
22
  COGNITE_SPACES,
21
23
  DMS_CONTAINER_PROPERTY_SIZE_LIMIT,
22
24
  DMS_VIEW_CONTAINER_SIZE_LIMIT,
25
+ get_base_concepts,
23
26
  )
24
- from cognite.neat.core._data_model.models.data_types import DataType
25
- from cognite.neat.core._data_model.models.entities import ContainerEntity, RawFilter
26
- from cognite.neat.core._data_model.models.entities._single_value import (
27
+ from cognite.neat.v0.core._data_model.models._import_contexts import ImportContext, SpreadsheetContext
28
+ from cognite.neat.v0.core._data_model.models.data_types import DataType
29
+ from cognite.neat.v0.core._data_model.models.entities import ContainerEntity, RawFilter
30
+ from cognite.neat.v0.core._data_model.models.entities._single_value import (
31
+ ContainerIndexEntity,
27
32
  ViewEntity,
28
33
  )
29
- from cognite.neat.core._issues import IssueList, NeatError
30
- from cognite.neat.core._issues.errors import (
34
+ from cognite.neat.v0.core._issues import IssueList, NeatError
35
+ from cognite.neat.v0.core._issues.errors import (
31
36
  CDFMissingClientError,
32
37
  PropertyDefinitionDuplicatedError,
38
+ PropertyInvalidDefinitionError,
33
39
  PropertyMappingDuplicatedError,
34
40
  PropertyNotFoundError,
35
41
  ResourceDuplicatedError,
36
42
  ResourceNotFoundError,
37
43
  ReversedConnectionNotFeasibleError,
38
44
  )
39
- from cognite.neat.core._issues.errors._external import CDFMissingResourcesError
40
- from cognite.neat.core._issues.warnings import (
45
+ from cognite.neat.v0.core._issues.errors._external import CDFMissingResourcesError
46
+ from cognite.neat.v0.core._issues.warnings import (
41
47
  NotSupportedHasDataFilterLimitWarning,
42
48
  NotSupportedViewContainerLimitWarning,
43
49
  ReversedConnectionNotFeasibleWarning,
44
50
  UndefinedViewWarning,
45
51
  user_modeling,
46
52
  )
47
- from cognite.neat.core._issues.warnings.user_modeling import (
53
+ from cognite.neat.v0.core._issues.warnings._models import ViewWithoutPropertiesWarning
54
+ from cognite.neat.v0.core._issues.warnings.user_modeling import (
48
55
  ContainerPropertyLimitWarning,
49
56
  DirectRelationMissingSourceWarning,
50
57
  NotNeatSupportedFilterWarning,
51
58
  )
52
- from cognite.neat.core._utils.spreadsheet import SpreadsheetRead
53
- from cognite.neat.core._utils.text import humanize_collection
59
+ from cognite.neat.v0.core._utils.text import humanize_collection
54
60
 
55
61
  from ._verified import PhysicalDataModel, PhysicalProperty
56
62
 
57
63
 
64
+ @dataclass
65
+ class _ContainerPropertyIndex:
66
+ """This is a helper class used in the indices validation
67
+
68
+ Args:
69
+ location: The index of the property in the properties list.
70
+ property_: The physical property associated with the container.
71
+ index: The index entity that defines the container property.
72
+ """
73
+
74
+ location: int
75
+ property_: PhysicalProperty
76
+ index: ContainerIndexEntity
77
+
78
+
58
79
  class PhysicalValidation:
59
80
  """This class does all the validation of the physical data model that
60
81
  have dependencies between components."""
@@ -63,15 +84,23 @@ class PhysicalValidation:
63
84
  self,
64
85
  data_model: PhysicalDataModel,
65
86
  client: NeatClient | None = None,
66
- read_info_by_spreadsheet: dict[str, SpreadsheetRead] | None = None,
87
+ context: ImportContext | None = None,
67
88
  ) -> None:
89
+ # import here to avoid circular import issues
90
+ from cognite.neat.v0.core._data_model.analysis._base import DataModelAnalysis
91
+
68
92
  self._data_model = data_model
69
93
  self._client = client
70
94
  self._metadata = data_model.metadata
71
95
  self._properties = data_model.properties
72
96
  self._containers = data_model.containers
73
97
  self._views = data_model.views
74
- self._read_info_by_spreadsheet = read_info_by_spreadsheet or {}
98
+ self._read_info_by_spreadsheet = context if isinstance(context, SpreadsheetContext) else SpreadsheetContext()
99
+
100
+ self.analysis = DataModelAnalysis(physical=self._data_model)
101
+ self._cdf_concepts = {
102
+ ViewEntity.load(concept_as_string) for concept_as_string in get_base_concepts(base_model="CogniteCore")
103
+ }
75
104
 
76
105
  def imported_views_and_containers_ids(
77
106
  self, include_views_with_no_properties: bool = True
@@ -93,9 +122,9 @@ class PhysicalValidation:
93
122
  view_with_properties.add(prop.view)
94
123
 
95
124
  for container in self._containers or []:
96
- for required in container.constraint or []:
97
- if required not in existing_containers:
98
- imported_containers.add(required)
125
+ for constraint in container.constraint or []:
126
+ if constraint.require not in existing_containers:
127
+ imported_containers.add(cast(ContainerEntity, constraint.require))
99
128
 
100
129
  if include_views_with_no_properties:
101
130
  extra_views = existing_views - view_with_properties
@@ -145,16 +174,21 @@ class PhysicalValidation:
145
174
  all_properties_by_ids = {**ref_properties_by_ids, **properties_by_ids}
146
175
  view_properties_by_id = self._as_view_properties_by_id(properties_by_ids)
147
176
  parents_view_ids_by_child_id = self._parent_view_ids_by_child_id(all_views_by_id)
177
+ container_properties_by_id = self._create_container_properties_by_id()
148
178
 
149
179
  issue_list = IssueList()
150
180
 
151
181
  # Validated for duplicated resource
152
182
  issue_list.extend(self._duplicated_resources())
153
183
 
184
+ # Validate if views are defined (i.e. have at least one property defined, or inherited)
185
+ issue_list.extend(self._views_without_properties_exist())
186
+
154
187
  # Neat DMS classes Validation
155
188
  # These are errors that can only happen due to the format of the Neat DMS classes
156
189
  issue_list.extend(self._validate_raw_filter())
157
- issue_list.extend(self._consistent_container_properties())
190
+ issue_list.extend(self._consistent_container_properties(container_properties_by_id))
191
+ issue_list.extend(self._valid_composite_container_indices(container_properties_by_id))
158
192
  issue_list.extend(self._validate_value_type_existence())
159
193
  issue_list.extend(
160
194
  self._validate_property_referenced_views_and_containers_exists(all_views_by_id, all_containers_by_id)
@@ -172,6 +206,23 @@ class PhysicalValidation:
172
206
  issue_list.extend(self._same_space_views_and_data_model())
173
207
  return issue_list
174
208
 
209
+ def _views_without_properties_exist(self) -> IssueList:
210
+ """Check if there are views that do not have any properties defined directly or inherited."""
211
+ issue_list = IssueList()
212
+ views = {view.view for view in self._views}
213
+ ancestors_by_view = self.analysis.implements_by_view(include_ancestors=True, include_different_space=True)
214
+ views_with_properties = self.analysis.defined_views().union(self._cdf_concepts)
215
+
216
+ if candidate_views := views.difference(views_with_properties):
217
+ for view in candidate_views:
218
+ # Here we check if at least one of the ancestors of the view has properties
219
+ if (ancestors := ancestors_by_view.get(view)) and ancestors.intersection(views_with_properties):
220
+ continue
221
+
222
+ issue_list.append_if_not_exist(ViewWithoutPropertiesWarning(view_id=view.as_id()))
223
+
224
+ return issue_list
225
+
175
226
  def _same_space_views_and_data_model(self) -> IssueList:
176
227
  issue_list = IssueList()
177
228
 
@@ -320,13 +371,27 @@ class PhysicalValidation:
320
371
  parents_by_view[view_id] = get_parents(view_id)
321
372
  return parents_by_view
322
373
 
323
- def _consistent_container_properties(self) -> IssueList:
374
+ def _create_container_properties_by_id(
375
+ self,
376
+ ) -> dict[tuple[ContainerEntity, str], list[tuple[int, PhysicalProperty]]]:
377
+ """Create a mapping of container properties with their location in the properties list.
378
+
379
+ Returns:
380
+ dict[tuple[ContainerEntity, str], list[tuple[int, PhysicalProperty]]]: A dictionary where the key is a tuple
381
+ of (ContainerEntity, property name) and the value is a list of tuples of (int, PhysicalProperty) where
382
+ int is the index of the property in the properties list and PhysicalProperty is the property itself.
383
+ """
324
384
  container_properties_by_id: dict[tuple[ContainerEntity, str], list[tuple[int, PhysicalProperty]]] = defaultdict(
325
385
  list
326
386
  )
327
387
  for prop_no, prop in enumerate(self._properties):
328
388
  if prop.container and prop.container_property:
329
389
  container_properties_by_id[(prop.container, prop.container_property)].append((prop_no, prop))
390
+ return container_properties_by_id
391
+
392
+ def _consistent_container_properties(
393
+ self, container_properties_by_id: dict[tuple[ContainerEntity, str], list[tuple[int, PhysicalProperty]]]
394
+ ) -> IssueList:
330
395
  properties_sheet = self._read_info_by_spreadsheet.get("Properties")
331
396
  errors = IssueList()
332
397
  for (container, prop_name), properties in container_properties_by_id.items():
@@ -392,11 +457,11 @@ class PhysicalValidation:
392
457
  )
393
458
  )
394
459
  index_definitions = {
395
- ",".join([index.suffix for index in prop.index]) for _, prop in properties if prop.index is not None
460
+ ",".join([str(index) for index in prop.index]) for _, prop in properties if prop.index is not None
396
461
  }
397
462
  if len(index_definitions) > 1:
398
463
  errors.append(
399
- PropertyDefinitionDuplicatedError[dm.ContainerId](
464
+ PropertyDefinitionDuplicatedError(
400
465
  container_id,
401
466
  "container",
402
467
  prop_name,
@@ -406,8 +471,11 @@ class PhysicalValidation:
406
471
  )
407
472
  )
408
473
  constraint_definitions = {
409
- ",".join(prop.constraint) for _, prop in properties if prop.constraint is not None
474
+ ",".join([str(constraint) for constraint in prop.constraint])
475
+ for _, prop in properties
476
+ if prop.constraint is not None
410
477
  }
478
+
411
479
  if len(constraint_definitions) > 1:
412
480
  errors.append(
413
481
  PropertyDefinitionDuplicatedError[dm.ContainerId](
@@ -422,6 +490,104 @@ class PhysicalValidation:
422
490
 
423
491
  return errors
424
492
 
493
+ def _valid_composite_container_indices(
494
+ self, container_properties_by_id: dict[tuple[ContainerEntity, str], list[tuple[int, PhysicalProperty]]]
495
+ ) -> IssueList:
496
+ """Validate that the indices on the container properties are valid."""
497
+ index_properties_by_container_index: dict[tuple[ContainerEntity, str], list[_ContainerPropertyIndex]] = (
498
+ defaultdict(list)
499
+ )
500
+ for (container, _), properties in container_properties_by_id.items():
501
+ for row_no, prop in properties:
502
+ for index in prop.index or []:
503
+ index_properties_by_container_index[(container, index.suffix)].append(
504
+ _ContainerPropertyIndex(row_no, prop, index)
505
+ )
506
+
507
+ properties_sheet_info = self._read_info_by_spreadsheet.get("Properties")
508
+ errors = IssueList()
509
+ for (container, _), index_properties in index_properties_by_container_index.items():
510
+ if len(index_properties) <= 1:
511
+ # If there is only one property in the index, this is already validated in the field_validator
512
+ # of the PhysicalProperty class. This validation is only for composite indices.
513
+ continue
514
+ container_id = container.as_id()
515
+ row_numbers = tuple([index_prop.location for index_prop in index_properties])
516
+ if properties_sheet_info:
517
+ row_numbers = tuple([properties_sheet_info.adjusted_row_number(row_no) for row_no in row_numbers])
518
+
519
+ if order_missing_error := self._validate_container_indices_has_order(
520
+ index_properties, row_numbers, container_id
521
+ ):
522
+ errors.append(order_missing_error)
523
+ same_order_errors = self._validate_container_indices_same_order(index_properties, row_numbers, container_id)
524
+ errors.extend(same_order_errors)
525
+ return errors
526
+
527
+ @staticmethod
528
+ def _validate_container_indices_has_order(
529
+ index_properties: list[_ContainerPropertyIndex], row_numbers: tuple[int, ...], container_id: dm.ContainerId
530
+ ) -> PropertyInvalidDefinitionError | None:
531
+ property_names: list[str] = []
532
+ indices: list[ContainerIndexEntity] = []
533
+ for prop_index in index_properties:
534
+ if prop_index.index.order is None:
535
+ property_names.append(prop_index.property_.view_property)
536
+ indices.append(prop_index.index)
537
+
538
+ if not property_names:
539
+ return None
540
+
541
+ if len(set(property_names)) == 1:
542
+ # If this is the same property, this is not a composite index, but a poorly defined single property
543
+ # index. This will be caught by the PropertyDefinitionDuplicatedError.
544
+ return None
545
+
546
+ properties_str = humanize_collection(property_names)
547
+ fixed_indices = [str(index.model_copy(update={"order": no})) for no, index in enumerate(indices, 1)]
548
+ message = (
549
+ "You must specify the order when using a composite index. "
550
+ f"For example {humanize_collection(fixed_indices)}."
551
+ )
552
+ return PropertyInvalidDefinitionError(
553
+ container_id,
554
+ "container",
555
+ properties_str,
556
+ message,
557
+ row_numbers,
558
+ "rows",
559
+ )
560
+
561
+ @staticmethod
562
+ def _validate_container_indices_same_order(
563
+ index_properties: list[_ContainerPropertyIndex], row_numbers: tuple[int, ...], container_id: dm.ContainerId
564
+ ) -> list[PropertyInvalidDefinitionError]:
565
+ """Checks whether there are multiple properties with the same order in a composite index."""
566
+ properties_by_order: dict[int, list[tuple[int, PhysicalProperty]]] = defaultdict(list)
567
+ for index_prop in index_properties:
568
+ if index_prop.index.order is not None:
569
+ properties_by_order[index_prop.index.order].append((index_prop.location, index_prop.property_))
570
+ same_order_errors: list[PropertyInvalidDefinitionError] = []
571
+ for order, props in properties_by_order.items():
572
+ if len(props) > 1:
573
+ properties_str = humanize_collection([prop.view_property for _, prop in props])
574
+ message = (
575
+ "You cannot have multiple properties with the same order in a composite index. "
576
+ f"Got order={order} for all composite properties."
577
+ "Please ensure that each property has an unique order."
578
+ )
579
+ same_order_errors.append(
580
+ PropertyInvalidDefinitionError(
581
+ container_id,
582
+ "container",
583
+ properties_str,
584
+ message,
585
+ row_numbers,
586
+ "rows",
587
+ )
588
+ )
589
+ return same_order_errors
590
+
425
591
  @staticmethod
426
592
  def _containers_are_proper_size(dms_schema: DMSSchema) -> IssueList:
427
593
  errors = IssueList()