cognite-toolkit 0.5.44__py3-none-any.whl → 0.7.34__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (807) hide show
  1. cognite_toolkit/_cdf.py +24 -26
  2. cognite_toolkit/_cdf_tk/apps/__init__.py +10 -2
  3. cognite_toolkit/_cdf_tk/apps/_core_app.py +30 -16
  4. cognite_toolkit/_cdf_tk/apps/_data_app.py +24 -0
  5. cognite_toolkit/_cdf_tk/apps/_dev_app.py +86 -0
  6. cognite_toolkit/_cdf_tk/apps/_download_app.py +1146 -0
  7. cognite_toolkit/_cdf_tk/apps/_dump_app.py +421 -102
  8. cognite_toolkit/_cdf_tk/apps/_import_app.py +41 -0
  9. cognite_toolkit/_cdf_tk/apps/_landing_app.py +17 -5
  10. cognite_toolkit/_cdf_tk/apps/_migrate_app.py +1085 -1
  11. cognite_toolkit/_cdf_tk/apps/_modules_app.py +37 -13
  12. cognite_toolkit/_cdf_tk/apps/_profile_app.py +142 -3
  13. cognite_toolkit/_cdf_tk/apps/_purge.py +250 -17
  14. cognite_toolkit/_cdf_tk/apps/_run.py +21 -10
  15. cognite_toolkit/_cdf_tk/apps/_upload_app.py +107 -0
  16. cognite_toolkit/_cdf_tk/builders/__init__.py +8 -4
  17. cognite_toolkit/_cdf_tk/builders/_base.py +37 -49
  18. cognite_toolkit/_cdf_tk/builders/_datamodels.py +4 -4
  19. cognite_toolkit/_cdf_tk/builders/_file.py +5 -5
  20. cognite_toolkit/_cdf_tk/builders/_function.py +7 -7
  21. cognite_toolkit/_cdf_tk/builders/_location.py +11 -31
  22. cognite_toolkit/_cdf_tk/builders/_raw.py +11 -11
  23. cognite_toolkit/_cdf_tk/builders/_streamlit.py +4 -4
  24. cognite_toolkit/_cdf_tk/builders/_transformation.py +11 -8
  25. cognite_toolkit/_cdf_tk/cdf_toml.py +77 -9
  26. cognite_toolkit/_cdf_tk/client/__init__.py +2 -1
  27. cognite_toolkit/_cdf_tk/client/_constants.py +1 -0
  28. cognite_toolkit/_cdf_tk/client/_toolkit_client.py +49 -65
  29. cognite_toolkit/_cdf_tk/client/api/canvas.py +185 -0
  30. cognite_toolkit/_cdf_tk/client/api/charts.py +125 -0
  31. cognite_toolkit/_cdf_tk/client/api/extended_data_modeling.py +70 -14
  32. cognite_toolkit/_cdf_tk/client/api/extended_files.py +227 -0
  33. cognite_toolkit/_cdf_tk/client/api/extended_functions.py +80 -0
  34. cognite_toolkit/_cdf_tk/client/api/extended_raw.py +49 -0
  35. cognite_toolkit/_cdf_tk/client/api/extended_timeseries.py +70 -2
  36. cognite_toolkit/_cdf_tk/client/api/fixed_transformations.py +122 -0
  37. cognite_toolkit/_cdf_tk/client/api/infield.py +254 -0
  38. cognite_toolkit/_cdf_tk/client/api/location_filters.py +19 -3
  39. cognite_toolkit/_cdf_tk/client/api/lookup.py +202 -71
  40. cognite_toolkit/_cdf_tk/client/api/migration.py +529 -0
  41. cognite_toolkit/_cdf_tk/client/api/project.py +24 -0
  42. cognite_toolkit/_cdf_tk/client/api/robotics/api.py +0 -2
  43. cognite_toolkit/_cdf_tk/client/api/robotics/capabilities.py +0 -2
  44. cognite_toolkit/_cdf_tk/client/api/robotics/data_postprocessing.py +0 -2
  45. cognite_toolkit/_cdf_tk/client/api/robotics/frames.py +0 -2
  46. cognite_toolkit/_cdf_tk/client/api/robotics/locations.py +0 -2
  47. cognite_toolkit/_cdf_tk/client/api/robotics/maps.py +0 -2
  48. cognite_toolkit/_cdf_tk/client/api/robotics/robots.py +0 -2
  49. cognite_toolkit/_cdf_tk/client/api/search.py +13 -0
  50. cognite_toolkit/_cdf_tk/client/api/search_config.py +49 -0
  51. cognite_toolkit/_cdf_tk/client/api/streams.py +89 -0
  52. cognite_toolkit/_cdf_tk/client/api/three_d.py +131 -0
  53. cognite_toolkit/_cdf_tk/client/api/token.py +111 -0
  54. cognite_toolkit/_cdf_tk/client/config.py +120 -0
  55. cognite_toolkit/_cdf_tk/client/data_classes/api_classes.py +17 -0
  56. cognite_toolkit/_cdf_tk/client/data_classes/base.py +71 -0
  57. cognite_toolkit/_cdf_tk/client/data_classes/canvas.py +1107 -0
  58. cognite_toolkit/_cdf_tk/client/data_classes/capabilities.py +170 -0
  59. cognite_toolkit/_cdf_tk/client/data_classes/charts.py +114 -0
  60. cognite_toolkit/_cdf_tk/client/data_classes/charts_data.py +184 -0
  61. cognite_toolkit/_cdf_tk/client/data_classes/extendable_cognite_file.py +8 -4
  62. cognite_toolkit/_cdf_tk/client/data_classes/extended_filemetadata.py +108 -0
  63. cognite_toolkit/_cdf_tk/client/data_classes/extended_filemetdata.py +113 -0
  64. cognite_toolkit/_cdf_tk/client/data_classes/functions.py +0 -2
  65. cognite_toolkit/_cdf_tk/client/data_classes/graphql_data_models.py +9 -5
  66. cognite_toolkit/_cdf_tk/client/data_classes/infield.py +116 -0
  67. cognite_toolkit/_cdf_tk/client/data_classes/instance_api.py +162 -0
  68. cognite_toolkit/_cdf_tk/client/data_classes/instances.py +24 -2
  69. cognite_toolkit/_cdf_tk/client/data_classes/location_filters.py +3 -5
  70. cognite_toolkit/_cdf_tk/client/data_classes/migration.py +457 -0
  71. cognite_toolkit/_cdf_tk/client/data_classes/project.py +41 -0
  72. cognite_toolkit/_cdf_tk/client/data_classes/raw.py +378 -9
  73. cognite_toolkit/_cdf_tk/client/data_classes/robotics.py +8 -10
  74. cognite_toolkit/_cdf_tk/client/data_classes/search_config.py +224 -0
  75. cognite_toolkit/_cdf_tk/client/data_classes/streamlit_.py +11 -11
  76. cognite_toolkit/_cdf_tk/client/data_classes/streams.py +90 -0
  77. cognite_toolkit/_cdf_tk/client/data_classes/three_d.py +53 -0
  78. cognite_toolkit/_cdf_tk/client/testing.py +83 -6
  79. cognite_toolkit/_cdf_tk/client/utils/_concurrency.py +24 -1
  80. cognite_toolkit/_cdf_tk/commands/__init__.py +20 -8
  81. cognite_toolkit/_cdf_tk/commands/_base.py +8 -3
  82. cognite_toolkit/_cdf_tk/commands/_changes.py +12 -46
  83. cognite_toolkit/_cdf_tk/commands/_cli_commands.py +0 -2
  84. cognite_toolkit/_cdf_tk/commands/_download.py +154 -0
  85. cognite_toolkit/_cdf_tk/{prototypes/commands/import_.py → commands/_import_cmd.py} +0 -2
  86. cognite_toolkit/_cdf_tk/commands/_migrate/__init__.py +7 -0
  87. cognite_toolkit/_cdf_tk/commands/_migrate/command.py +309 -0
  88. cognite_toolkit/_cdf_tk/commands/_migrate/conversion.py +373 -0
  89. cognite_toolkit/_cdf_tk/commands/_migrate/creators.py +232 -0
  90. cognite_toolkit/_cdf_tk/commands/_migrate/data_classes.py +294 -0
  91. cognite_toolkit/_cdf_tk/commands/_migrate/data_mapper.py +471 -0
  92. cognite_toolkit/_cdf_tk/commands/_migrate/data_model.py +229 -0
  93. cognite_toolkit/_cdf_tk/commands/_migrate/default_mappings.py +137 -0
  94. cognite_toolkit/_cdf_tk/commands/_migrate/issues.py +212 -0
  95. cognite_toolkit/_cdf_tk/commands/_migrate/migration_io.py +456 -0
  96. cognite_toolkit/_cdf_tk/commands/_migrate/prepare.py +60 -0
  97. cognite_toolkit/_cdf_tk/commands/_migrate/selectors.py +71 -0
  98. cognite_toolkit/_cdf_tk/commands/_profile.py +921 -149
  99. cognite_toolkit/_cdf_tk/commands/_purge.py +692 -511
  100. cognite_toolkit/_cdf_tk/commands/_questionary_style.py +16 -0
  101. cognite_toolkit/_cdf_tk/commands/_upload.py +289 -0
  102. cognite_toolkit/_cdf_tk/commands/_utils.py +1 -26
  103. cognite_toolkit/_cdf_tk/commands/_virtual_env.py +0 -2
  104. cognite_toolkit/_cdf_tk/commands/about.py +221 -0
  105. cognite_toolkit/_cdf_tk/commands/auth.py +46 -26
  106. cognite_toolkit/_cdf_tk/commands/build_cmd.py +72 -114
  107. cognite_toolkit/_cdf_tk/commands/build_v2/build_cmd.py +241 -0
  108. cognite_toolkit/_cdf_tk/commands/build_v2/build_input.py +85 -0
  109. cognite_toolkit/_cdf_tk/commands/build_v2/build_issues.py +27 -0
  110. cognite_toolkit/_cdf_tk/commands/clean.py +89 -39
  111. cognite_toolkit/_cdf_tk/commands/deploy.py +334 -194
  112. cognite_toolkit/_cdf_tk/commands/dump_resource.py +470 -79
  113. cognite_toolkit/_cdf_tk/commands/init.py +219 -18
  114. cognite_toolkit/_cdf_tk/commands/modules.py +319 -71
  115. cognite_toolkit/_cdf_tk/commands/pull.py +45 -46
  116. cognite_toolkit/_cdf_tk/commands/repo.py +1 -1
  117. cognite_toolkit/_cdf_tk/commands/resources.py +179 -0
  118. cognite_toolkit/_cdf_tk/commands/run.py +11 -13
  119. cognite_toolkit/_cdf_tk/constants.py +104 -8
  120. cognite_toolkit/_cdf_tk/cruds/__init__.py +224 -0
  121. cognite_toolkit/_cdf_tk/{loaders/_base_loaders.py → cruds/_base_cruds.py} +72 -134
  122. cognite_toolkit/_cdf_tk/{loaders/_data_loaders.py → cruds/_data_cruds.py} +29 -38
  123. cognite_toolkit/_cdf_tk/cruds/_resource_cruds/__init__.py +98 -0
  124. cognite_toolkit/_cdf_tk/cruds/_resource_cruds/agent.py +115 -0
  125. cognite_toolkit/_cdf_tk/{loaders/_resource_loaders/auth_loaders.py → cruds/_resource_cruds/auth.py} +33 -70
  126. cognite_toolkit/_cdf_tk/{loaders/_resource_loaders/classic_loaders.py → cruds/_resource_cruds/classic.py} +86 -122
  127. cognite_toolkit/_cdf_tk/cruds/_resource_cruds/configuration.py +149 -0
  128. cognite_toolkit/_cdf_tk/{loaders/_resource_loaders/data_organization_loaders.py → cruds/_resource_cruds/data_organization.py} +17 -50
  129. cognite_toolkit/_cdf_tk/{loaders/_resource_loaders/datamodel_loaders.py → cruds/_resource_cruds/datamodel.py} +280 -372
  130. cognite_toolkit/_cdf_tk/{loaders/_resource_loaders/extraction_pipeline_loaders.py → cruds/_resource_cruds/extraction_pipeline.py} +70 -89
  131. cognite_toolkit/_cdf_tk/cruds/_resource_cruds/fieldops.py +438 -0
  132. cognite_toolkit/_cdf_tk/{loaders/_resource_loaders/file_loader.py → cruds/_resource_cruds/file.py} +28 -83
  133. cognite_toolkit/_cdf_tk/{loaders/_resource_loaders/function_loaders.py → cruds/_resource_cruds/function.py} +179 -108
  134. cognite_toolkit/_cdf_tk/{loaders/_resource_loaders/group_scoped_loader.py → cruds/_resource_cruds/group_scoped.py} +19 -19
  135. cognite_toolkit/_cdf_tk/{loaders/_resource_loaders → cruds/_resource_cruds}/hosted_extractors.py +32 -102
  136. cognite_toolkit/_cdf_tk/{loaders/_resource_loaders/industrial_tool_loaders.py → cruds/_resource_cruds/industrial_tool.py} +15 -27
  137. cognite_toolkit/_cdf_tk/{loaders/_resource_loaders/location_loaders.py → cruds/_resource_cruds/location.py} +64 -170
  138. cognite_toolkit/_cdf_tk/cruds/_resource_cruds/migration.py +111 -0
  139. cognite_toolkit/_cdf_tk/{loaders/_resource_loaders/raw_loaders.py → cruds/_resource_cruds/raw.py} +21 -26
  140. cognite_toolkit/_cdf_tk/{loaders/_resource_loaders/relationship_loader.py → cruds/_resource_cruds/relationship.py} +21 -43
  141. cognite_toolkit/_cdf_tk/{loaders/_resource_loaders/robotics_loaders.py → cruds/_resource_cruds/robotics.py} +29 -50
  142. cognite_toolkit/_cdf_tk/cruds/_resource_cruds/streams.py +94 -0
  143. cognite_toolkit/_cdf_tk/{loaders/_resource_loaders/three_d_model_loaders.py → cruds/_resource_cruds/three_d_model.py} +11 -33
  144. cognite_toolkit/_cdf_tk/cruds/_resource_cruds/timeseries.py +500 -0
  145. cognite_toolkit/_cdf_tk/{loaders/_resource_loaders/transformation_loaders.py → cruds/_resource_cruds/transformation.py} +155 -144
  146. cognite_toolkit/_cdf_tk/{loaders/_resource_loaders/workflow_loaders.py → cruds/_resource_cruds/workflow.py} +116 -203
  147. cognite_toolkit/_cdf_tk/{loaders → cruds}/_worker.py +85 -78
  148. cognite_toolkit/_cdf_tk/data_classes/__init__.py +2 -0
  149. cognite_toolkit/_cdf_tk/data_classes/_base.py +8 -4
  150. cognite_toolkit/_cdf_tk/data_classes/_build_files.py +3 -3
  151. cognite_toolkit/_cdf_tk/data_classes/_build_variables.py +133 -23
  152. cognite_toolkit/_cdf_tk/data_classes/_built_modules.py +23 -9
  153. cognite_toolkit/_cdf_tk/data_classes/_built_resources.py +24 -21
  154. cognite_toolkit/_cdf_tk/data_classes/_config_yaml.py +34 -26
  155. cognite_toolkit/_cdf_tk/data_classes/_deploy_results.py +26 -7
  156. cognite_toolkit/_cdf_tk/data_classes/_module_directories.py +37 -18
  157. cognite_toolkit/_cdf_tk/data_classes/_module_resources.py +12 -10
  158. cognite_toolkit/_cdf_tk/data_classes/_module_toml.py +12 -10
  159. cognite_toolkit/_cdf_tk/data_classes/_packages.py +48 -24
  160. cognite_toolkit/_cdf_tk/data_classes/_tracking_info.py +43 -0
  161. cognite_toolkit/_cdf_tk/exceptions.py +32 -3
  162. cognite_toolkit/_cdf_tk/feature_flags.py +59 -46
  163. cognite_toolkit/_cdf_tk/hints.py +5 -5
  164. cognite_toolkit/_cdf_tk/plugins.py +2 -3
  165. cognite_toolkit/_cdf_tk/protocols.py +97 -0
  166. cognite_toolkit/_cdf_tk/resource_classes/__init__.py +63 -0
  167. cognite_toolkit/_cdf_tk/resource_classes/agent.py +58 -0
  168. cognite_toolkit/_cdf_tk/resource_classes/agent_tools.py +101 -0
  169. cognite_toolkit/_cdf_tk/resource_classes/authentication.py +15 -0
  170. cognite_toolkit/_cdf_tk/resource_classes/capabilities.py +30 -1
  171. cognite_toolkit/_cdf_tk/resource_classes/cognitefile.py +51 -0
  172. cognite_toolkit/_cdf_tk/resource_classes/container_field_definitions.py +326 -0
  173. cognite_toolkit/_cdf_tk/resource_classes/containers.py +92 -0
  174. cognite_toolkit/_cdf_tk/resource_classes/data_model.py +51 -0
  175. cognite_toolkit/_cdf_tk/resource_classes/datapoint_subscription.py +77 -0
  176. cognite_toolkit/_cdf_tk/resource_classes/extraction_pipeline.py +54 -0
  177. cognite_toolkit/_cdf_tk/resource_classes/extraction_pipeline_config.py +15 -0
  178. cognite_toolkit/_cdf_tk/resource_classes/function_schedule.py +3 -7
  179. cognite_toolkit/_cdf_tk/resource_classes/functions.py +12 -1
  180. cognite_toolkit/_cdf_tk/resource_classes/graphql_model.py +50 -0
  181. cognite_toolkit/_cdf_tk/resource_classes/hosted_extractor_destination.py +19 -0
  182. cognite_toolkit/_cdf_tk/resource_classes/hosted_extractor_job.py +255 -0
  183. cognite_toolkit/_cdf_tk/resource_classes/hosted_extractor_mapping.py +117 -0
  184. cognite_toolkit/_cdf_tk/resource_classes/hosted_extractor_source.py +390 -0
  185. cognite_toolkit/_cdf_tk/resource_classes/infield_cdm_location_config.py +109 -0
  186. cognite_toolkit/_cdf_tk/resource_classes/infield_location_config.py +94 -0
  187. cognite_toolkit/_cdf_tk/resource_classes/infield_v1.py +100 -0
  188. cognite_toolkit/_cdf_tk/resource_classes/instance.py +93 -0
  189. cognite_toolkit/_cdf_tk/resource_classes/location.py +1 -1
  190. cognite_toolkit/_cdf_tk/resource_classes/migration.py +28 -0
  191. cognite_toolkit/_cdf_tk/resource_classes/raw_database_table.py +6 -2
  192. cognite_toolkit/_cdf_tk/resource_classes/relationship.py +50 -0
  193. cognite_toolkit/_cdf_tk/resource_classes/robotics/__init__.py +13 -0
  194. cognite_toolkit/_cdf_tk/resource_classes/robotics/capability.py +19 -0
  195. cognite_toolkit/_cdf_tk/resource_classes/robotics/data_postprocessing.py +15 -0
  196. cognite_toolkit/_cdf_tk/resource_classes/robotics/frame.py +25 -0
  197. cognite_toolkit/_cdf_tk/resource_classes/robotics/location.py +12 -0
  198. cognite_toolkit/_cdf_tk/resource_classes/robotics/map.py +16 -0
  199. cognite_toolkit/_cdf_tk/resource_classes/search_config.py +30 -0
  200. cognite_toolkit/_cdf_tk/resource_classes/sequence.py +107 -0
  201. cognite_toolkit/_cdf_tk/resource_classes/space.py +4 -4
  202. cognite_toolkit/_cdf_tk/resource_classes/streamlit_.py +26 -0
  203. cognite_toolkit/_cdf_tk/resource_classes/streams.py +29 -0
  204. cognite_toolkit/_cdf_tk/resource_classes/transformation_destination.py +1 -1
  205. cognite_toolkit/_cdf_tk/resource_classes/transformation_notification.py +8 -0
  206. cognite_toolkit/_cdf_tk/resource_classes/transformations.py +2 -13
  207. cognite_toolkit/_cdf_tk/resource_classes/view_field_definitions.py +207 -0
  208. cognite_toolkit/_cdf_tk/resource_classes/views.py +93 -0
  209. cognite_toolkit/_cdf_tk/resource_classes/workflow.py +15 -0
  210. cognite_toolkit/_cdf_tk/resource_classes/workflow_trigger.py +115 -0
  211. cognite_toolkit/_cdf_tk/resource_classes/workflow_version.py +233 -0
  212. cognite_toolkit/_cdf_tk/storageio/__init__.py +69 -0
  213. cognite_toolkit/_cdf_tk/storageio/_annotations.py +105 -0
  214. cognite_toolkit/_cdf_tk/storageio/_applications.py +408 -0
  215. cognite_toolkit/_cdf_tk/storageio/_asset_centric.py +693 -0
  216. cognite_toolkit/_cdf_tk/storageio/_base.py +328 -0
  217. cognite_toolkit/_cdf_tk/storageio/_data_classes.py +91 -0
  218. cognite_toolkit/_cdf_tk/storageio/_datapoints.py +448 -0
  219. cognite_toolkit/_cdf_tk/storageio/_file_content.py +436 -0
  220. cognite_toolkit/_cdf_tk/storageio/_instances.py +225 -0
  221. cognite_toolkit/_cdf_tk/storageio/_raw.py +124 -0
  222. cognite_toolkit/_cdf_tk/storageio/selectors/__init__.py +133 -0
  223. cognite_toolkit/_cdf_tk/storageio/selectors/_asset_centric.py +60 -0
  224. cognite_toolkit/_cdf_tk/storageio/selectors/_base.py +80 -0
  225. cognite_toolkit/_cdf_tk/storageio/selectors/_canvas.py +22 -0
  226. cognite_toolkit/_cdf_tk/storageio/selectors/_charts.py +45 -0
  227. cognite_toolkit/_cdf_tk/storageio/selectors/_datapoints.py +89 -0
  228. cognite_toolkit/_cdf_tk/storageio/selectors/_file_content.py +164 -0
  229. cognite_toolkit/_cdf_tk/storageio/selectors/_instances.py +162 -0
  230. cognite_toolkit/_cdf_tk/storageio/selectors/_raw.py +24 -0
  231. cognite_toolkit/_cdf_tk/storageio/selectors/_three_d.py +34 -0
  232. cognite_toolkit/_cdf_tk/tk_warnings/__init__.py +2 -0
  233. cognite_toolkit/_cdf_tk/tk_warnings/base.py +1 -3
  234. cognite_toolkit/_cdf_tk/tk_warnings/fileread.py +8 -6
  235. cognite_toolkit/_cdf_tk/tk_warnings/other.py +18 -3
  236. cognite_toolkit/_cdf_tk/tracker.py +21 -24
  237. cognite_toolkit/_cdf_tk/utils/__init__.py +4 -6
  238. cognite_toolkit/_cdf_tk/utils/_auxiliary.py +36 -0
  239. cognite_toolkit/_cdf_tk/utils/aggregators.py +424 -0
  240. cognite_toolkit/_cdf_tk/utils/auth.py +4 -2
  241. cognite_toolkit/_cdf_tk/utils/auxiliary.py +23 -0
  242. cognite_toolkit/_cdf_tk/utils/cdf.py +191 -21
  243. cognite_toolkit/_cdf_tk/utils/cli_args.py +34 -0
  244. cognite_toolkit/_cdf_tk/utils/collection.py +53 -2
  245. cognite_toolkit/_cdf_tk/utils/dtype_conversion.py +593 -0
  246. cognite_toolkit/_cdf_tk/utils/file.py +55 -2
  247. cognite_toolkit/_cdf_tk/utils/fileio/__init__.py +61 -0
  248. cognite_toolkit/_cdf_tk/utils/fileio/_base.py +32 -0
  249. cognite_toolkit/_cdf_tk/utils/fileio/_compression.py +61 -0
  250. cognite_toolkit/_cdf_tk/utils/fileio/_readers.py +422 -0
  251. cognite_toolkit/_cdf_tk/utils/fileio/_writers.py +429 -0
  252. cognite_toolkit/_cdf_tk/utils/graphql_parser.py +7 -2
  253. cognite_toolkit/_cdf_tk/utils/hashing.py +3 -9
  254. cognite_toolkit/_cdf_tk/utils/http_client/__init__.py +69 -0
  255. cognite_toolkit/_cdf_tk/utils/http_client/_client.py +536 -0
  256. cognite_toolkit/_cdf_tk/utils/http_client/_data_classes.py +412 -0
  257. cognite_toolkit/_cdf_tk/utils/http_client/_data_classes2.py +249 -0
  258. cognite_toolkit/_cdf_tk/utils/http_client/_exception.py +4 -0
  259. cognite_toolkit/_cdf_tk/utils/http_client/_tracker.py +34 -0
  260. cognite_toolkit/_cdf_tk/utils/interactive_select.py +819 -102
  261. cognite_toolkit/_cdf_tk/utils/modules.py +10 -14
  262. cognite_toolkit/_cdf_tk/utils/producer_worker.py +280 -63
  263. cognite_toolkit/_cdf_tk/utils/progress_tracker.py +87 -0
  264. cognite_toolkit/_cdf_tk/utils/sentry_utils.py +3 -3
  265. cognite_toolkit/_cdf_tk/utils/sql_parser.py +160 -0
  266. cognite_toolkit/_cdf_tk/utils/text.py +115 -0
  267. cognite_toolkit/_cdf_tk/utils/thread_safe_dict.py +99 -0
  268. cognite_toolkit/_cdf_tk/utils/useful_types.py +31 -0
  269. cognite_toolkit/_cdf_tk/utils/validate_access.py +468 -0
  270. cognite_toolkit/_cdf_tk/validation.py +172 -80
  271. cognite_toolkit/_repo_files/.gitignore +2 -0
  272. cognite_toolkit/_repo_files/AzureDevOps/.devops/deploy-pipeline.yml +1 -1
  273. cognite_toolkit/_repo_files/AzureDevOps/.devops/dry-run-pipeline.yml +1 -1
  274. cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml +1 -1
  275. cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml +1 -1
  276. cognite_toolkit/_resources/cdf.toml +18 -0
  277. cognite_toolkit/_version.py +1 -1
  278. cognite_toolkit/config.dev.yaml +13 -0
  279. cognite_toolkit/demo/_base.py +4 -4
  280. {cognite_toolkit-0.5.44.dist-info → cognite_toolkit-0.7.34.dist-info}/METADATA +25 -21
  281. cognite_toolkit-0.7.34.dist-info/RECORD +320 -0
  282. cognite_toolkit-0.7.34.dist-info/WHEEL +4 -0
  283. {cognite_toolkit-0.5.44.dist-info → cognite_toolkit-0.7.34.dist-info}/entry_points.txt +1 -0
  284. cognite_toolkit/_builtin_modules/README.md +0 -7
  285. cognite_toolkit/_builtin_modules/bootcamp/README.md +0 -5
  286. cognite_toolkit/_builtin_modules/bootcamp/default.config.yaml +0 -3
  287. cognite_toolkit/_builtin_modules/bootcamp/ice_cream_api/default.config.yaml +0 -9
  288. cognite_toolkit/_builtin_modules/bootcamp/ice_cream_api/functions/icapi_datapoints_extractor/handler.py +0 -168
  289. cognite_toolkit/_builtin_modules/bootcamp/ice_cream_api/functions/icapi_datapoints_extractor/ice_cream_factory_api.py +0 -61
  290. cognite_toolkit/_builtin_modules/bootcamp/ice_cream_api/functions/icapi_datapoints_extractor/requirements.txt +0 -3
  291. cognite_toolkit/_builtin_modules/bootcamp/ice_cream_api/module.toml +0 -8
  292. cognite_toolkit/_builtin_modules/bootcamp/module.toml +0 -8
  293. cognite_toolkit/_builtin_modules/bootcamp/use_cases/oee/default.config.yaml +0 -6
  294. cognite_toolkit/_builtin_modules/bootcamp/use_cases/oee/functions/oee_timeseries/handler.py +0 -229
  295. cognite_toolkit/_builtin_modules/bootcamp/use_cases/oee/functions/oee_timeseries/requirements.txt +0 -2
  296. cognite_toolkit/_builtin_modules/bootcamp/use_cases/oee/module.toml +0 -8
  297. cognite_toolkit/_builtin_modules/cdf.toml +0 -13
  298. cognite_toolkit/_builtin_modules/cdf_common/data_models/demo.Space.yaml +0 -3
  299. cognite_toolkit/_builtin_modules/cdf_common/data_sets/demo.DataSet.yaml +0 -3
  300. cognite_toolkit/_builtin_modules/cdf_common/default.config.yaml +0 -8
  301. cognite_toolkit/_builtin_modules/cdf_common/extraction_pipelines/ctx_files_direct_relation_write.ExtractionPipeline.yaml +0 -54
  302. cognite_toolkit/_builtin_modules/cdf_common/extraction_pipelines/ctx_files_direct_relation_write.config.yaml +0 -28
  303. cognite_toolkit/_builtin_modules/cdf_common/functions/contextualization_connection_writer/handler.py +0 -342
  304. cognite_toolkit/_builtin_modules/cdf_common/functions/contextualization_connection_writer/requirements.txt +0 -3
  305. cognite_toolkit/_builtin_modules/cdf_common/functions/write.Function.yaml +0 -7
  306. cognite_toolkit/_builtin_modules/cdf_common/module.toml +0 -7
  307. cognite_toolkit/_builtin_modules/cdf_common/raw/contextualization_state.DataBase.yaml +0 -1
  308. cognite_toolkit/_builtin_modules/cdf_common/raw/contextualization_state.Table.yaml +0 -2
  309. cognite_toolkit/_builtin_modules/cdf_common/raw/source.DataBase.yaml +0 -1
  310. cognite_toolkit/_builtin_modules/cdf_ingestion/auth/user.Group.yaml +0 -124
  311. cognite_toolkit/_builtin_modules/cdf_ingestion/auth/workflow.Group.yaml +0 -114
  312. cognite_toolkit/_builtin_modules/cdf_ingestion/default.config.yaml +0 -34
  313. cognite_toolkit/_builtin_modules/cdf_ingestion/module.toml +0 -14
  314. cognite_toolkit/_builtin_modules/cdf_ingestion/workflows/ingest.Workflow.yaml +0 -3
  315. cognite_toolkit/_builtin_modules/cdf_ingestion/workflows/trigger.WorkflowTrigger.yaml +0 -9
  316. cognite_toolkit/_builtin_modules/cdf_ingestion/workflows/v1.WorkflowVersion.yaml +0 -150
  317. cognite_toolkit/_builtin_modules/common/README.md +0 -5
  318. cognite_toolkit/_builtin_modules/common/cdf_apm_base/data_models/apm.datamodel.yaml +0 -18
  319. cognite_toolkit/_builtin_modules/common/cdf_apm_base/data_models/apm_config.datamodel.yaml +0 -10
  320. cognite_toolkit/_builtin_modules/common/cdf_apm_base/data_models/config.space.yaml +0 -3
  321. cognite_toolkit/_builtin_modules/common/cdf_apm_base/data_models/containers/APM_Activity.container.yaml +0 -108
  322. cognite_toolkit/_builtin_modules/common/cdf_apm_base/data_models/containers/APM_Notification.container.yaml +0 -85
  323. cognite_toolkit/_builtin_modules/common/cdf_apm_base/data_models/containers/APM_Operation.container.yaml +0 -83
  324. cognite_toolkit/_builtin_modules/common/cdf_apm_base/data_models/containers/ApmConfig.container.yaml +0 -39
  325. cognite_toolkit/_builtin_modules/common/cdf_apm_base/data_models/source.space.yaml +0 -3
  326. cognite_toolkit/_builtin_modules/common/cdf_apm_base/data_models/views/APM_Activity.view.yaml +0 -119
  327. cognite_toolkit/_builtin_modules/common/cdf_apm_base/data_models/views/APM_Notification.view.yaml +0 -84
  328. cognite_toolkit/_builtin_modules/common/cdf_apm_base/data_models/views/APM_Operation.view.yaml +0 -86
  329. cognite_toolkit/_builtin_modules/common/cdf_apm_base/data_models/views/ApmConfig.view.yaml +0 -48
  330. cognite_toolkit/_builtin_modules/common/cdf_apm_base/default.config.yaml +0 -1
  331. cognite_toolkit/_builtin_modules/common/cdf_apm_base/module.toml +0 -9
  332. cognite_toolkit/_builtin_modules/common/cdf_auth_readwrite_all/README.md +0 -23
  333. cognite_toolkit/_builtin_modules/common/cdf_auth_readwrite_all/auth/admin.readonly.group.yaml +0 -260
  334. cognite_toolkit/_builtin_modules/common/cdf_auth_readwrite_all/auth/admin.readwrite.group.yaml +0 -323
  335. cognite_toolkit/_builtin_modules/common/cdf_auth_readwrite_all/default.config.yaml +0 -2
  336. cognite_toolkit/_builtin_modules/common/cdf_auth_readwrite_all/module.toml +0 -7
  337. cognite_toolkit/_builtin_modules/contextualization/cdf_connection_sql/default.config.yaml +0 -10
  338. cognite_toolkit/_builtin_modules/contextualization/cdf_connection_sql/module.toml +0 -19
  339. cognite_toolkit/_builtin_modules/contextualization/cdf_connection_sql/transformations/activity_to_asset.Transformation.sql +0 -31
  340. cognite_toolkit/_builtin_modules/contextualization/cdf_connection_sql/transformations/activity_to_asset.Transformation.yaml +0 -13
  341. cognite_toolkit/_builtin_modules/contextualization/cdf_connection_sql/transformations/activity_to_equipment.Transformation.sql +0 -31
  342. cognite_toolkit/_builtin_modules/contextualization/cdf_connection_sql/transformations/activity_to_equipment.Transformation.yaml +0 -14
  343. cognite_toolkit/_builtin_modules/contextualization/cdf_connection_sql/transformations/activity_to_timeseries.Transformation.sql +0 -34
  344. cognite_toolkit/_builtin_modules/contextualization/cdf_connection_sql/transformations/activity_to_timeseries.Transformation.yaml +0 -13
  345. cognite_toolkit/_builtin_modules/contextualization/cdf_connection_sql/transformations/timeseries_to_asset.Transformation.sql +0 -32
  346. cognite_toolkit/_builtin_modules/contextualization/cdf_connection_sql/transformations/timeseries_to_asset.Transformation.yaml +0 -14
  347. cognite_toolkit/_builtin_modules/contextualization/cdf_connection_sql/transformations/timeseries_to_equipment.Transformation.sql +0 -32
  348. cognite_toolkit/_builtin_modules/contextualization/cdf_connection_sql/transformations/timeseries_to_equipment.Transformation.yaml +0 -13
  349. cognite_toolkit/_builtin_modules/contextualization/cdf_entity_matching/auth/workflow.Group.yaml +0 -83
  350. cognite_toolkit/_builtin_modules/contextualization/cdf_entity_matching/data_models/annotation_type.Node.yaml +0 -2
  351. cognite_toolkit/_builtin_modules/contextualization/cdf_entity_matching/data_models/sourceSystem.node.yaml +0 -13
  352. cognite_toolkit/_builtin_modules/contextualization/cdf_entity_matching/default.config.yaml +0 -16
  353. cognite_toolkit/_builtin_modules/contextualization/cdf_entity_matching/extraction_pipelines/ctx_files_entity_matcher.ExtractionPipeline.yaml +0 -48
  354. cognite_toolkit/_builtin_modules/contextualization/cdf_entity_matching/extraction_pipelines/ctx_files_entity_matcher.config.yaml +0 -35
  355. cognite_toolkit/_builtin_modules/contextualization/cdf_entity_matching/functions/contextualization_entity_matcher/handler.py +0 -476
  356. cognite_toolkit/_builtin_modules/contextualization/cdf_entity_matching/functions/contextualization_entity_matcher/requirements.txt +0 -3
  357. cognite_toolkit/_builtin_modules/contextualization/cdf_entity_matching/functions/functions.Function.yaml +0 -7
  358. cognite_toolkit/_builtin_modules/contextualization/cdf_entity_matching/module.toml +0 -28
  359. cognite_toolkit/_builtin_modules/contextualization/cdf_entity_matching/raw/state.Table.yaml +0 -2
  360. cognite_toolkit/_builtin_modules/contextualization/cdf_entity_matching/workflows/entity_matching.Workflow.yaml +0 -3
  361. cognite_toolkit/_builtin_modules/contextualization/cdf_entity_matching/workflows/trigger.WorkflowTrigger.yaml +0 -9
  362. cognite_toolkit/_builtin_modules/contextualization/cdf_entity_matching/workflows/v1.WorkflowVersion.yaml +0 -22
  363. cognite_toolkit/_builtin_modules/contextualization/cdf_p_and_id_parser/auth/function.Group.yaml +0 -89
  364. cognite_toolkit/_builtin_modules/contextualization/cdf_p_and_id_parser/data_models/sourceSystem.node.yaml +0 -13
  365. cognite_toolkit/_builtin_modules/contextualization/cdf_p_and_id_parser/default.config.yaml +0 -13
  366. cognite_toolkit/_builtin_modules/contextualization/cdf_p_and_id_parser/extraction_pipelines/ctx_files_pandid_annotater.ExtractionPipeline.yaml +0 -44
  367. cognite_toolkit/_builtin_modules/contextualization/cdf_p_and_id_parser/extraction_pipelines/ctx_files_pandid_annotater.config.yaml +0 -33
  368. cognite_toolkit/_builtin_modules/contextualization/cdf_p_and_id_parser/functions/contextualization_p_and_id_annotater/handler.py +0 -405
  369. cognite_toolkit/_builtin_modules/contextualization/cdf_p_and_id_parser/functions/contextualization_p_and_id_annotater/requirements.txt +0 -3
  370. cognite_toolkit/_builtin_modules/contextualization/cdf_p_and_id_parser/functions/functions.Function.yaml +0 -7
  371. cognite_toolkit/_builtin_modules/contextualization/cdf_p_and_id_parser/module.toml +0 -31
  372. cognite_toolkit/_builtin_modules/contextualization/cdf_p_and_id_parser/workflows/p_and_id_parser.Workflow.yaml +0 -3
  373. cognite_toolkit/_builtin_modules/contextualization/cdf_p_and_id_parser/workflows/trigger.WorkflowTrigger.yaml +0 -9
  374. cognite_toolkit/_builtin_modules/contextualization/cdf_p_and_id_parser/workflows/v1.WorkflowVersion.yaml +0 -22
  375. cognite_toolkit/_builtin_modules/custom/my_module/auth/.gitkeep +0 -0
  376. cognite_toolkit/_builtin_modules/custom/my_module/default.config.yaml +0 -1
  377. cognite_toolkit/_builtin_modules/custom/my_module/module.toml +0 -7
  378. cognite_toolkit/_builtin_modules/industrial_tools/cdf_location_filter_asset_centric/classic/root.Asset.yaml +0 -9
  379. cognite_toolkit/_builtin_modules/industrial_tools/cdf_location_filter_asset_centric/data_sets/data_sets.DataSet.yaml +0 -3
  380. cognite_toolkit/_builtin_modules/industrial_tools/cdf_location_filter_asset_centric/default.config.yaml +0 -4
  381. cognite_toolkit/_builtin_modules/industrial_tools/cdf_location_filter_asset_centric/locations/individualFilters.LocationFilter.yaml +0 -19
  382. cognite_toolkit/_builtin_modules/industrial_tools/cdf_location_filter_asset_centric/locations/oneFilter.LocationFilter.yaml +0 -12
  383. cognite_toolkit/_builtin_modules/industrial_tools/cdf_location_filter_asset_centric/module.toml +0 -7
  384. cognite_toolkit/_builtin_modules/industrial_tools/cdf_location_filter_datamodel_based/data_models/instance.Space.yaml +0 -2
  385. cognite_toolkit/_builtin_modules/industrial_tools/cdf_location_filter_datamodel_based/default.config.yaml +0 -2
  386. cognite_toolkit/_builtin_modules/industrial_tools/cdf_location_filter_datamodel_based/locations/dataModel.LocationFilter.yaml +0 -17
  387. cognite_toolkit/_builtin_modules/industrial_tools/cdf_location_filter_datamodel_based/module.toml +0 -7
  388. cognite_toolkit/_builtin_modules/industrial_tools/cdf_search/auth/locationfilter.Group.yaml +0 -12
  389. cognite_toolkit/_builtin_modules/industrial_tools/cdf_search/default.config.yaml +0 -3
  390. cognite_toolkit/_builtin_modules/industrial_tools/cdf_search/locations/user.LocationFilter.yaml +0 -9
  391. cognite_toolkit/_builtin_modules/industrial_tools/cdf_search/module.toml +0 -11
  392. cognite_toolkit/_builtin_modules/infield/cdf_infield_common/README.md +0 -3
  393. cognite_toolkit/_builtin_modules/infield/cdf_infield_common/auth/applications-configuration.Group.yaml +0 -22
  394. cognite_toolkit/_builtin_modules/infield/cdf_infield_common/data_models/infieldAppData.space.yaml +0 -3
  395. cognite_toolkit/_builtin_modules/infield/cdf_infield_common/data_models/infield_apm_app_config.node.yaml +0 -28
  396. cognite_toolkit/_builtin_modules/infield/cdf_infield_common/default.config.yaml +0 -2
  397. cognite_toolkit/_builtin_modules/infield/cdf_infield_common/module.toml +0 -11
  398. cognite_toolkit/_builtin_modules/infield/cdf_infield_location/README.md +0 -61
  399. cognite_toolkit/_builtin_modules/infield/cdf_infield_location/auth/infield_checklist_admin_role.group.yaml +0 -100
  400. cognite_toolkit/_builtin_modules/infield/cdf_infield_location/auth/infield_normal_role.group.yaml +0 -100
  401. cognite_toolkit/_builtin_modules/infield/cdf_infield_location/auth/infield_template_admin_role.group.yaml +0 -100
  402. cognite_toolkit/_builtin_modules/infield/cdf_infield_location/auth/infield_viewer_role.group.yaml +0 -85
  403. cognite_toolkit/_builtin_modules/infield/cdf_infield_location/data_models/infieldLocationAppData.space.yaml +0 -3
  404. cognite_toolkit/_builtin_modules/infield/cdf_infield_location/data_models/infieldLocationSourceData.space.yaml +0 -3
  405. cognite_toolkit/_builtin_modules/infield/cdf_infield_location/data_sets/location_app_data_set.DataSet.yaml +0 -3
  406. cognite_toolkit/_builtin_modules/infield/cdf_infield_location/data_sets/location_source_data_set.DataSet.yaml +0 -20
  407. cognite_toolkit/_builtin_modules/infield/cdf_infield_location/default.config.yaml +0 -23
  408. cognite_toolkit/_builtin_modules/infield/cdf_infield_location/module.toml +0 -8
  409. cognite_toolkit/_builtin_modules/infield/cdf_infield_location/transformations/tr_asset_oid_workmate_infield_sync_assets_from_hierarchy_to_apm.Transformation.sql +0 -34
  410. cognite_toolkit/_builtin_modules/infield/cdf_infield_location/transformations/tr_asset_oid_workmate_infield_sync_assets_from_hierarchy_to_apm.Transformation.yaml +0 -24
  411. cognite_toolkit/_builtin_modules/infield/cdf_infield_location/transformations/tr_asset_oid_workmate_infield_sync_assets_from_hierarchy_to_apm.schedule.yaml +0 -3
  412. cognite_toolkit/_builtin_modules/infield/cdf_infield_location/transformations/tr_workorder_oid_workmate_infield_sync_workorders_to_apm_activities.Transformation.sql +0 -17
  413. cognite_toolkit/_builtin_modules/infield/cdf_infield_location/transformations/tr_workorder_oid_workmate_infield_sync_workorders_to_apm_activities.Transformation.yaml +0 -25
  414. cognite_toolkit/_builtin_modules/infield/cdf_infield_location/transformations/tr_workorder_oid_workmate_infield_sync_workorders_to_apm_activities.schedule.yaml +0 -3
  415. cognite_toolkit/_builtin_modules/infield/cdf_infield_second_location/README.md +0 -61
  416. cognite_toolkit/_builtin_modules/infield/cdf_infield_second_location/auth/infield_checklist_admin_role.group.yaml +0 -100
  417. cognite_toolkit/_builtin_modules/infield/cdf_infield_second_location/auth/infield_normal_role.group.yaml +0 -100
  418. cognite_toolkit/_builtin_modules/infield/cdf_infield_second_location/auth/infield_template_admin_role.group.yaml +0 -100
  419. cognite_toolkit/_builtin_modules/infield/cdf_infield_second_location/auth/infield_viewer_role.group.yaml +0 -85
  420. cognite_toolkit/_builtin_modules/infield/cdf_infield_second_location/data_models/infieldLocationAppData.space.yaml +0 -3
  421. cognite_toolkit/_builtin_modules/infield/cdf_infield_second_location/data_models/infieldLocationSourceData.space.yaml +0 -3
  422. cognite_toolkit/_builtin_modules/infield/cdf_infield_second_location/data_sets/location_app_data_set.DataSet.yaml +0 -3
  423. cognite_toolkit/_builtin_modules/infield/cdf_infield_second_location/data_sets/location_source_data_set.DataSet.yaml +0 -20
  424. cognite_toolkit/_builtin_modules/infield/cdf_infield_second_location/default.config.yaml +0 -23
  425. cognite_toolkit/_builtin_modules/infield/cdf_infield_second_location/module.toml +0 -8
  426. cognite_toolkit/_builtin_modules/infield/cdf_infield_second_location/transformations/tr_asset_oid_workmate_infield_sync_assets_from_hierarchy_to_apm.Transformation.sql +0 -34
  427. cognite_toolkit/_builtin_modules/infield/cdf_infield_second_location/transformations/tr_asset_oid_workmate_infield_sync_assets_from_hierarchy_to_apm.Transformation.yaml +0 -24
  428. cognite_toolkit/_builtin_modules/infield/cdf_infield_second_location/transformations/tr_asset_oid_workmate_infield_sync_assets_from_hierarchy_to_apm.schedule.yaml +0 -3
  429. cognite_toolkit/_builtin_modules/infield/cdf_infield_second_location/transformations/tr_workorder_oid_workmate_infield_sync_workorders_to_apm_activities.Transformation.sql +0 -17
  430. cognite_toolkit/_builtin_modules/infield/cdf_infield_second_location/transformations/tr_workorder_oid_workmate_infield_sync_workorders_to_apm_activities.Transformation.yaml +0 -25
  431. cognite_toolkit/_builtin_modules/infield/cdf_infield_second_location/transformations/tr_workorder_oid_workmate_infield_sync_workorders_to_apm_activities.schedule.yaml +0 -3
  432. cognite_toolkit/_builtin_modules/infield/default.config.yaml +0 -21
  433. cognite_toolkit/_builtin_modules/inrobot/README.md +0 -41
  434. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/README.md +0 -53
  435. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/auth/run-function-user.Group.yaml +0 -89
  436. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/classic/robot.Label.yaml +0 -6
  437. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/data_models/cogniteAppData.space.yaml +0 -3
  438. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/data_models/inrobotApmConfig.node.yaml +0 -17
  439. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/data_sets/robot_1.dataset.yaml +0 -2
  440. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/default.config.yaml +0 -3
  441. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/functions/.gitkeep +0 -0
  442. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/functions/fn_contextualize_robot_data/README.md +0 -30
  443. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/functions/fn_contextualize_robot_data/__init__.py +0 -1
  444. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/functions/fn_contextualize_robot_data/common/README.md +0 -58
  445. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/functions/fn_contextualize_robot_data/common/__init__.py +0 -0
  446. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/functions/fn_contextualize_robot_data/common/apm_helpers.py +0 -269
  447. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/functions/fn_contextualize_robot_data/common/cdf_helpers.py +0 -112
  448. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/functions/fn_contextualize_robot_data/common/dataclass/__init__.py +0 -0
  449. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/functions/fn_contextualize_robot_data/common/dataclass/common.py +0 -16
  450. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/functions/fn_contextualize_robot_data/common/dataclass/vision.py +0 -66
  451. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/functions/fn_contextualize_robot_data/common/utils.py +0 -17
  452. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/functions/fn_contextualize_robot_data/handler.py +0 -269
  453. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/functions/fn_contextualize_robot_data/requirements.txt +0 -22
  454. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/functions/fn_gauge_reading/README.md +0 -56
  455. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/functions/fn_gauge_reading/__init__.py +0 -1
  456. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/functions/fn_gauge_reading/common/README.md +0 -58
  457. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/functions/fn_gauge_reading/common/__init__.py +0 -0
  458. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/functions/fn_gauge_reading/common/apm_helpers.py +0 -269
  459. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/functions/fn_gauge_reading/common/cdf_helpers.py +0 -112
  460. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/functions/fn_gauge_reading/common/dataclass/__init__.py +0 -0
  461. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/functions/fn_gauge_reading/common/dataclass/common.py +0 -16
  462. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/functions/fn_gauge_reading/common/dataclass/vision.py +0 -66
  463. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/functions/fn_gauge_reading/common/utils.py +0 -17
  464. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/functions/fn_gauge_reading/handler.py +0 -396
  465. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/functions/fn_gauge_reading/requirements.txt +0 -22
  466. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/functions/fn_get_ir_data_from_ir_raw/README.md +0 -18
  467. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/functions/fn_get_ir_data_from_ir_raw/__init__.py +0 -1
  468. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/functions/fn_get_ir_data_from_ir_raw/common/README.md +0 -58
  469. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/functions/fn_get_ir_data_from_ir_raw/common/__init__.py +0 -0
  470. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/functions/fn_get_ir_data_from_ir_raw/common/apm_helpers.py +0 -269
  471. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/functions/fn_get_ir_data_from_ir_raw/common/cdf_helpers.py +0 -112
  472. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/functions/fn_get_ir_data_from_ir_raw/common/dataclass/__init__.py +0 -0
  473. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/functions/fn_get_ir_data_from_ir_raw/common/dataclass/common.py +0 -16
  474. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/functions/fn_get_ir_data_from_ir_raw/common/dataclass/vision.py +0 -66
  475. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/functions/fn_get_ir_data_from_ir_raw/common/utils.py +0 -17
  476. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/functions/fn_get_ir_data_from_ir_raw/handler.py +0 -195
  477. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/functions/fn_get_ir_data_from_ir_raw/requirements.txt +0 -23
  478. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/functions/fn_threesixty/README.md +0 -14
  479. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/functions/fn_threesixty/__init__.py +0 -1
  480. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/functions/fn_threesixty/cognite_threesixty_images.py +0 -284
  481. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/functions/fn_threesixty/handler.py +0 -257
  482. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/functions/fn_threesixty/requirements.txt +0 -7
  483. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/functions/robots.functions.Function.yaml +0 -51
  484. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/functions/schedules.Schedule.yaml +0 -94
  485. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/module.toml +0 -11
  486. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/robotics/acoustic_video.RobotCapability.yaml +0 -72
  487. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/robotics/process_threesixty.DataPostProcessing.yaml +0 -23
  488. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/robotics/pt_ir_video.RobotCapability.yaml +0 -64
  489. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/robotics/ptz.RobotCapability.yaml +0 -63
  490. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/robotics/ptz_ir.RobotCapability.yaml +0 -80
  491. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/robotics/ptz_video.RobotCapability.yaml +0 -68
  492. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/robotics/read_dial_gauge.DataPostProcessing.yaml +0 -31
  493. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/robotics/read_digital_gauge.DataPostProcessing.yaml +0 -35
  494. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/robotics/read_level_gauge.DataPostProcessing.yaml +0 -29
  495. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/robotics/read_valve.DataPostProcessing.yaml +0 -23
  496. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/robotics/threesixty.RobotCapability.yaml +0 -47
  497. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_common/robotics/threesixty_video.RobotCapability.yaml +0 -47
  498. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_location/README.md +0 -33
  499. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_location/auth/inrobot-admin.Group.yaml +0 -107
  500. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_location/auth/inrobot-users.Group.yaml +0 -94
  501. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_location/auth/robot-1.Group.yaml +0 -77
  502. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_location/data_models/locationAppData.space.yaml +0 -3
  503. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_location/data_models/locationSourceData.space.yaml +0 -3
  504. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_location/default.config.yaml +0 -7
  505. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_location/module.toml +0 -8
  506. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_location/robotics/root.Frame.yaml +0 -15
  507. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_location/robotics/root.Location.yaml +0 -3
  508. cognite_toolkit/_builtin_modules/inrobot/cdf_inrobot_location/robotics/root.Map.yaml +0 -10
  509. cognite_toolkit/_builtin_modules/inrobot/default.config.yaml +0 -3
  510. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension/data_models/containers/Activity.Container.yaml +0 -13
  511. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension/data_models/containers/Asset.Container.yaml +0 -13
  512. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension/data_models/containers/Equipment.Container.yaml +0 -13
  513. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension/data_models/containers/File.Container.yaml +0 -13
  514. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension/data_models/containers/TimeSeries.Container.yaml +0 -13
  515. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension/data_models/enterprise.datamodel.yaml +0 -54
  516. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension/data_models/schema.space.yaml +0 -1
  517. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension/data_models/views/Activity.view.yaml +0 -69
  518. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension/data_models/views/Asset.view.yaml +0 -186
  519. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension/data_models/views/Equipment.view.yaml +0 -100
  520. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension/data_models/views/File.view.yaml +0 -72
  521. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension/data_models/views/TimeSeries.view.yaml +0 -85
  522. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension/default.config.yaml +0 -2
  523. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension/module.toml +0 -7
  524. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/containers/360Image.Container.yaml +0 -13
  525. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/containers/360ImageAnnotation.Container.yaml +0 -13
  526. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/containers/360ImageCollection.Container.yaml +0 -13
  527. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/containers/360ImageModel.Container.yaml +0 -13
  528. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/containers/360ImageStation.Container.yaml +0 -13
  529. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/containers/3DModel.Container.yaml +0 -13
  530. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/containers/3DObject.Container.yaml +0 -13
  531. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/containers/3DRevision.Container.yaml +0 -13
  532. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/containers/3DTransformation.Container.yaml +0 -13
  533. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/containers/Activity.Container.yaml +0 -13
  534. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/containers/Annotation.Container.yaml +0 -14
  535. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/containers/Asset.Container.yaml +0 -13
  536. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/containers/AssetClass.Container.yaml +0 -13
  537. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/containers/AssetType.Container.yaml +0 -13
  538. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/containers/CADModel.Container.yaml +0 -13
  539. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/containers/CADNode.Container.yaml +0 -13
  540. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/containers/CADRevision.Container.yaml +0 -13
  541. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/containers/CubeMap.Container.yaml +0 -13
  542. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/containers/Describable.Container.yaml +0 -13
  543. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/containers/DiagramAnnotation.Container.yaml +0 -13
  544. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/containers/Equipment.Container.yaml +0 -13
  545. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/containers/EquipmentType.Container.yaml +0 -13
  546. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/containers/File.Container.yaml +0 -13
  547. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/containers/FileCategory.Container.yaml +0 -13
  548. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/containers/PointCloudModel.Container.yaml +0 -13
  549. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/containers/PointCloudRevision.Container.yaml +0 -13
  550. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/containers/PointCloudVolume.Container.yaml +0 -13
  551. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/containers/Schedulable.Container.yaml +0 -13
  552. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/containers/SourceSystem.Container.yaml +0 -13
  553. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/containers/Sourceable.Container.yaml +0 -13
  554. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/containers/TimeSeries.Container.yaml +0 -13
  555. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/containers/Unit.Container.yaml +0 -13
  556. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/containers/Visualizable.Container.yaml +0 -13
  557. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/enterprise.datamodel.yaml +0 -138
  558. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/schema.space.yaml +0 -1
  559. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/views/360Image.view.yaml +0 -119
  560. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/views/360ImageAnnotation.view.yaml +0 -28
  561. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/views/360ImageCollection.view.yaml +0 -39
  562. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/views/360ImageModel.view.yaml +0 -55
  563. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/views/360ImageStation.view.yaml +0 -29
  564. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/views/3DModel.view.yaml +0 -29
  565. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/views/3DObject.view.yaml +0 -81
  566. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/views/3DRevision.view.yaml +0 -30
  567. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/views/3DTransformation.view.yaml +0 -23
  568. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/views/Activity.view.yaml +0 -69
  569. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/views/Annotation.view.yaml +0 -28
  570. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/views/Asset.view.yaml +0 -191
  571. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/views/AssetClass.view.yaml +0 -17
  572. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/views/AssetType.view.yaml +0 -31
  573. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/views/CADModel.view.yaml +0 -55
  574. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/views/CADNode.view.yaml +0 -53
  575. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/views/CADRevision.view.yaml +0 -39
  576. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/views/CubeMap.view.yaml +0 -97
  577. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/views/Describable.view.yaml +0 -19
  578. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/views/DiagramAnnotation.view.yaml +0 -28
  579. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/views/Equipment.view.yaml +0 -100
  580. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/views/EquipmentType.view.yaml +0 -17
  581. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/views/File.view.yaml +0 -72
  582. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/views/FileCategory.view.yaml +0 -18
  583. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/views/PointCloudModel.view.yaml +0 -55
  584. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/views/PointCloudRevision.view.yaml +0 -39
  585. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/views/PointCloudVolume.view.yaml +0 -52
  586. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/views/Schedulable.view.yaml +0 -17
  587. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/views/SourceSystem.view.yaml +0 -17
  588. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/views/Sourceable.view.yaml +0 -27
  589. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/views/TimeSeries.view.yaml +0 -85
  590. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/views/Unit.view.yaml +0 -16
  591. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/data_models/views/Visualizable.view.yaml +0 -29
  592. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/default.config.yaml +0 -2
  593. cognite_toolkit/_builtin_modules/models/cdf_cdm_extension_full/module.toml +0 -7
  594. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension/data_models/containers/Activity.Container.yaml +0 -13
  595. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension/data_models/containers/Asset.Container.yaml +0 -13
  596. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension/data_models/containers/Equipment.Container.yaml +0 -13
  597. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension/data_models/containers/File.Container.yaml +0 -13
  598. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension/data_models/containers/MaintenanceOrder.Container.yaml +0 -13
  599. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension/data_models/containers/Notification.Container.yaml +0 -13
  600. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension/data_models/containers/Operation.Container.yaml +0 -13
  601. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension/data_models/containers/TimeSeries.Container.yaml +0 -13
  602. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension/data_models/enterprise.datamodel.yaml +0 -66
  603. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension/data_models/schema.space.yaml +0 -1
  604. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension/data_models/views/Activity.view.yaml +0 -69
  605. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension/data_models/views/Asset.view.yaml +0 -186
  606. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension/data_models/views/Equipment.view.yaml +0 -100
  607. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension/data_models/views/File.view.yaml +0 -72
  608. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension/data_models/views/MaintenanceOrder.view.yaml +0 -100
  609. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension/data_models/views/Notification.view.yaml +0 -56
  610. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension/data_models/views/Operation.view.yaml +0 -96
  611. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension/data_models/views/TimeSeries.view.yaml +0 -85
  612. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension/default.config.yaml +0 -3
  613. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension/module.toml +0 -8
  614. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/containers/360Image.Container.yaml +0 -13
  615. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/containers/360ImageAnnotation.Container.yaml +0 -14
  616. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/containers/360ImageCollection.Container.yaml +0 -13
  617. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/containers/360ImageModel.Container.yaml +0 -13
  618. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/containers/360ImageStation.Container.yaml +0 -13
  619. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/containers/3DModel.Container.yaml +0 -13
  620. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/containers/3DObject.Container.yaml +0 -13
  621. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/containers/3DRevision.Container.yaml +0 -13
  622. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/containers/3DTransformation.Container.yaml +0 -13
  623. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/containers/Activity.Container.yaml +0 -13
  624. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/containers/Annotation.Container.yaml +0 -14
  625. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/containers/Asset.Container.yaml +0 -13
  626. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/containers/AssetClass.Container.yaml +0 -13
  627. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/containers/AssetType.Container.yaml +0 -13
  628. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/containers/CADModel.Container.yaml +0 -13
  629. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/containers/CADNode.Container.yaml +0 -13
  630. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/containers/CADRevision.Container.yaml +0 -13
  631. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/containers/CubeMap.Container.yaml +0 -13
  632. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/containers/Describable.Container.yaml +0 -13
  633. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/containers/DiagramAnnotation.Container.yaml +0 -14
  634. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/containers/Equipment.Container.yaml +0 -13
  635. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/containers/EquipmentType.Container.yaml +0 -13
  636. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/containers/File.Container.yaml +0 -13
  637. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/containers/FileCategory.Container.yaml +0 -13
  638. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/containers/MaintenanceOrder.Container.yaml +0 -13
  639. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/containers/Notification.Container.yaml +0 -13
  640. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/containers/Operation.Container.yaml +0 -13
  641. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/containers/PointCloudModel.Container.yaml +0 -13
  642. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/containers/PointCloudRevision.Container.yaml +0 -13
  643. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/containers/PointCloudVolume.Container.yaml +0 -13
  644. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/containers/Schedulable.Container.yaml +0 -13
  645. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/containers/SourceSystem.Container.yaml +0 -13
  646. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/containers/Sourceable.Container.yaml +0 -13
  647. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/containers/TimeSeries.Container.yaml +0 -13
  648. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/containers/Unit.Container.yaml +0 -13
  649. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/containers/Visualizable.Container.yaml +0 -13
  650. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/enterprise.datamodel.yaml +0 -150
  651. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/schema.space.yaml +0 -1
  652. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/views/360Image.view.yaml +0 -119
  653. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/views/360ImageAnnotation.view.yaml +0 -28
  654. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/views/360ImageCollection.view.yaml +0 -39
  655. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/views/360ImageModel.view.yaml +0 -55
  656. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/views/360ImageStation.view.yaml +0 -29
  657. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/views/3DModel.view.yaml +0 -29
  658. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/views/3DObject.view.yaml +0 -81
  659. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/views/3DRevision.view.yaml +0 -30
  660. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/views/3DTransformation.view.yaml +0 -23
  661. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/views/Activity.view.yaml +0 -69
  662. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/views/Annotation.view.yaml +0 -28
  663. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/views/Asset.view.yaml +0 -191
  664. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/views/AssetClass.view.yaml +0 -17
  665. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/views/AssetType.view.yaml +0 -31
  666. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/views/CADModel.view.yaml +0 -55
  667. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/views/CADNode.view.yaml +0 -53
  668. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/views/CADRevision.view.yaml +0 -39
  669. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/views/CubeMap.view.yaml +0 -97
  670. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/views/Describable.view.yaml +0 -19
  671. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/views/DiagramAnnotation.view.yaml +0 -28
  672. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/views/Equipment.view.yaml +0 -100
  673. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/views/EquipmentType.view.yaml +0 -17
  674. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/views/File.view.yaml +0 -72
  675. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/views/FileCategory.view.yaml +0 -18
  676. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/views/MaintenanceOrder.view.yaml +0 -100
  677. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/views/Notification.view.yaml +0 -56
  678. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/views/Operation.view.yaml +0 -96
  679. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/views/PointCloudModel.view.yaml +0 -55
  680. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/views/PointCloudRevision.view.yaml +0 -39
  681. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/views/PointCloudVolume.view.yaml +0 -52
  682. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/views/Schedulable.view.yaml +0 -17
  683. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/views/SourceSystem.view.yaml +0 -17
  684. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/views/Sourceable.view.yaml +0 -27
  685. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/views/TimeSeries.view.yaml +0 -85
  686. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/views/Unit.view.yaml +0 -16
  687. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/data_models/views/Visualizable.view.yaml +0 -29
  688. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/default.config.yaml +0 -2
  689. cognite_toolkit/_builtin_modules/models/cdf_process_industry_extension_full/module.toml +0 -7
  690. cognite_toolkit/_builtin_modules/models/cdf_scene/data_models/containers/scene_Cdf3dRevisionProperties.Container.yaml +0 -13
  691. cognite_toolkit/_builtin_modules/models/cdf_scene/data_models/containers/scene_EnvironmentMap.Container.yaml +0 -12
  692. cognite_toolkit/_builtin_modules/models/cdf_scene/data_models/containers/scene_Image360CollectionProperties.Container.yaml +0 -23
  693. cognite_toolkit/_builtin_modules/models/cdf_scene/data_models/containers/scene_Image360CollectionScene.Container.yaml +0 -12
  694. cognite_toolkit/_builtin_modules/models/cdf_scene/data_models/containers/scene_SceneConfiguration.Container.yaml +0 -152
  695. cognite_toolkit/_builtin_modules/models/cdf_scene/data_models/containers/scene_TexturedMap.Container.yaml +0 -21
  696. cognite_toolkit/_builtin_modules/models/cdf_scene/data_models/containers/scene_TexturedPlane.Container.yaml +0 -29
  697. cognite_toolkit/_builtin_modules/models/cdf_scene/data_models/scene.Space.yaml +0 -3
  698. cognite_toolkit/_builtin_modules/models/cdf_scene/data_models/scene_data_model.DataModel.yaml +0 -66
  699. cognite_toolkit/_builtin_modules/models/cdf_scene/data_models/views/EnvironmentMap.View.yaml +0 -28
  700. cognite_toolkit/_builtin_modules/models/cdf_scene/data_models/views/Image360CollectionProperties.View.yaml +0 -22
  701. cognite_toolkit/_builtin_modules/models/cdf_scene/data_models/views/Image360CollectionScene.View.yaml +0 -20
  702. cognite_toolkit/_builtin_modules/models/cdf_scene/data_models/views/RevisionProperties.View.yaml +0 -16
  703. cognite_toolkit/_builtin_modules/models/cdf_scene/data_models/views/SceneConfiguration.View.yaml +0 -166
  704. cognite_toolkit/_builtin_modules/models/cdf_scene/data_models/views/TexturedMap.View.yaml +0 -17
  705. cognite_toolkit/_builtin_modules/models/cdf_scene/data_models/views/TexturedPlane.View.yaml +0 -40
  706. cognite_toolkit/_builtin_modules/models/cdf_scene/default.config.yaml +0 -1
  707. cognite_toolkit/_builtin_modules/models/cdf_scene/module.toml +0 -5
  708. cognite_toolkit/_builtin_modules/package.toml +0 -41
  709. cognite_toolkit/_builtin_modules/sourcesystem/README.md +0 -4
  710. cognite_toolkit/_builtin_modules/sourcesystem/cdf_pi/auth/workflow.Group.yaml +0 -53
  711. cognite_toolkit/_builtin_modules/sourcesystem/cdf_pi/default.config.yaml +0 -13
  712. cognite_toolkit/_builtin_modules/sourcesystem/cdf_pi/extraction_pipelines/extractor.Config.yaml +0 -5
  713. cognite_toolkit/_builtin_modules/sourcesystem/cdf_pi/extraction_pipelines/extractor.ExtractionPipeline.yaml +0 -12
  714. cognite_toolkit/_builtin_modules/sourcesystem/cdf_pi/module.toml +0 -30
  715. cognite_toolkit/_builtin_modules/sourcesystem/cdf_pi/raw/timeseries.Table.yaml +0 -2
  716. cognite_toolkit/_builtin_modules/sourcesystem/cdf_pi/transformations/population/timeseries.Transformation.sql +0 -9
  717. cognite_toolkit/_builtin_modules/sourcesystem/cdf_pi/transformations/population/timeseries.Transformation.yaml +0 -14
  718. cognite_toolkit/_builtin_modules/sourcesystem/cdf_pi/workflows/population.Workflow.yaml +0 -3
  719. cognite_toolkit/_builtin_modules/sourcesystem/cdf_pi/workflows/trigger.WorkflowTrigger.yaml +0 -9
  720. cognite_toolkit/_builtin_modules/sourcesystem/cdf_pi/workflows/v1.WorkflowVersion.yaml +0 -12
  721. cognite_toolkit/_builtin_modules/sourcesystem/cdf_sap_assets/auth/workflow.Group.yaml +0 -54
  722. cognite_toolkit/_builtin_modules/sourcesystem/cdf_sap_assets/default.config.yaml +0 -16
  723. cognite_toolkit/_builtin_modules/sourcesystem/cdf_sap_assets/module.toml +0 -30
  724. cognite_toolkit/_builtin_modules/sourcesystem/cdf_sap_assets/raw/dump.Table.yaml +0 -2
  725. cognite_toolkit/_builtin_modules/sourcesystem/cdf_sap_assets/transformations/connection/equipment_to_asset.Transformation.sql +0 -14
  726. cognite_toolkit/_builtin_modules/sourcesystem/cdf_sap_assets/transformations/connection/equipment_to_asset.Transformation.yaml +0 -14
  727. cognite_toolkit/_builtin_modules/sourcesystem/cdf_sap_assets/transformations/population/asset.Transformation.sql +0 -35
  728. cognite_toolkit/_builtin_modules/sourcesystem/cdf_sap_assets/transformations/population/asset.Transformation.yaml +0 -14
  729. cognite_toolkit/_builtin_modules/sourcesystem/cdf_sap_assets/transformations/population/equipment.Transformation.sql +0 -17
  730. cognite_toolkit/_builtin_modules/sourcesystem/cdf_sap_assets/transformations/population/equipment.Transformation.yaml +0 -13
  731. cognite_toolkit/_builtin_modules/sourcesystem/cdf_sap_assets/workflows/population.Workflow.yaml +0 -3
  732. cognite_toolkit/_builtin_modules/sourcesystem/cdf_sap_assets/workflows/trigger.WorkflowTrigger.yaml +0 -9
  733. cognite_toolkit/_builtin_modules/sourcesystem/cdf_sap_assets/workflows/v1.WorkflowVersion.yaml +0 -28
  734. cognite_toolkit/_builtin_modules/sourcesystem/cdf_sap_events/auth/workflow.Group.yaml +0 -46
  735. cognite_toolkit/_builtin_modules/sourcesystem/cdf_sap_events/default.config.yaml +0 -15
  736. cognite_toolkit/_builtin_modules/sourcesystem/cdf_sap_events/module.toml +0 -26
  737. cognite_toolkit/_builtin_modules/sourcesystem/cdf_sap_events/raw/workitem.Table.yaml +0 -2
  738. cognite_toolkit/_builtin_modules/sourcesystem/cdf_sap_events/raw/workorder.Table.yaml +0 -2
  739. cognite_toolkit/_builtin_modules/sourcesystem/cdf_sap_events/raw/workpackage.Table.yaml +0 -2
  740. cognite_toolkit/_builtin_modules/sourcesystem/cdf_sap_events/raw/worktask.Table.yaml +0 -3
  741. cognite_toolkit/_builtin_modules/sourcesystem/cdf_sap_events/transformations/connection/operation_to_maintenance_order.Transformation.sql +0 -32
  742. cognite_toolkit/_builtin_modules/sourcesystem/cdf_sap_events/transformations/connection/operation_to_maintenance_order.Transformation.yaml +0 -13
  743. cognite_toolkit/_builtin_modules/sourcesystem/cdf_sap_events/transformations/population/maintenanceOrder.Transformation.sql +0 -18
  744. cognite_toolkit/_builtin_modules/sourcesystem/cdf_sap_events/transformations/population/maintenanceOrder.Transformation.yaml +0 -14
  745. cognite_toolkit/_builtin_modules/sourcesystem/cdf_sap_events/transformations/population/operation.Transformation.sql +0 -18
  746. cognite_toolkit/_builtin_modules/sourcesystem/cdf_sap_events/transformations/population/operation.Transformation.yaml +0 -14
  747. cognite_toolkit/_builtin_modules/sourcesystem/cdf_sap_events/workflows/population.Workflow.yaml +0 -3
  748. cognite_toolkit/_builtin_modules/sourcesystem/cdf_sap_events/workflows/trigger.WorkflowTrigger.yaml +0 -9
  749. cognite_toolkit/_builtin_modules/sourcesystem/cdf_sap_events/workflows/v1.WorkflowVersion.yaml +0 -28
  750. cognite_toolkit/_builtin_modules/sourcesystem/cdf_sharepoint/auth/workflow.Group.yaml +0 -47
  751. cognite_toolkit/_builtin_modules/sourcesystem/cdf_sharepoint/default.config.yaml +0 -13
  752. cognite_toolkit/_builtin_modules/sourcesystem/cdf_sharepoint/files/upload.CogniteFile.yaml +0 -4
  753. cognite_toolkit/_builtin_modules/sourcesystem/cdf_sharepoint/module.toml +0 -32
  754. cognite_toolkit/_builtin_modules/sourcesystem/cdf_sharepoint/raw/files.Table.yaml +0 -2
  755. cognite_toolkit/_builtin_modules/sourcesystem/cdf_sharepoint/transformations/population/files.Transformation.sql +0 -9
  756. cognite_toolkit/_builtin_modules/sourcesystem/cdf_sharepoint/transformations/population/files.Transformation.yaml +0 -13
  757. cognite_toolkit/_builtin_modules/sourcesystem/cdf_sharepoint/workflows/population.Workflow.yaml +0 -3
  758. cognite_toolkit/_builtin_modules/sourcesystem/cdf_sharepoint/workflows/trigger.WorkflowTrigger.yaml +0 -9
  759. cognite_toolkit/_builtin_modules/sourcesystem/cdf_sharepoint/workflows/v1.WorkflowVersion.yaml +0 -11
  760. cognite_toolkit/_cdf_tk/_parameters/__init__.py +0 -18
  761. cognite_toolkit/_cdf_tk/_parameters/constants.py +0 -82
  762. cognite_toolkit/_cdf_tk/_parameters/data_classes.py +0 -197
  763. cognite_toolkit/_cdf_tk/_parameters/functions.py +0 -192
  764. cognite_toolkit/_cdf_tk/_parameters/get_type_hints.py +0 -125
  765. cognite_toolkit/_cdf_tk/_parameters/type_hint.py +0 -118
  766. cognite_toolkit/_cdf_tk/apps/_populate_app.py +0 -80
  767. cognite_toolkit/_cdf_tk/client/api/agents/agents.py +0 -107
  768. cognite_toolkit/_cdf_tk/client/api/agents/api.py +0 -12
  769. cognite_toolkit/_cdf_tk/client/api/statistics.py +0 -72
  770. cognite_toolkit/_cdf_tk/client/data_classes/agent_tools.py +0 -92
  771. cognite_toolkit/_cdf_tk/client/data_classes/agents.py +0 -135
  772. cognite_toolkit/_cdf_tk/client/data_classes/statistics.py +0 -130
  773. cognite_toolkit/_cdf_tk/commands/_populate.py +0 -306
  774. cognite_toolkit/_cdf_tk/commands/dump_data.py +0 -487
  775. cognite_toolkit/_cdf_tk/commands/featureflag.py +0 -29
  776. cognite_toolkit/_cdf_tk/loaders/__init__.py +0 -204
  777. cognite_toolkit/_cdf_tk/loaders/_resource_loaders/__init__.py +0 -90
  778. cognite_toolkit/_cdf_tk/loaders/_resource_loaders/agent_loaders.py +0 -89
  779. cognite_toolkit/_cdf_tk/loaders/_resource_loaders/fieldops_loaders.py +0 -260
  780. cognite_toolkit/_cdf_tk/loaders/_resource_loaders/timeseries_loaders.py +0 -356
  781. cognite_toolkit/_cdf_tk/prototypes/_packages/__init__.py +0 -1
  782. cognite_toolkit/_cdf_tk/prototypes/_packages/reference/manifest.yaml +0 -4
  783. cognite_toolkit/_cdf_tk/prototypes/_packages/reference/references/auth/reference.Group.yaml +0 -375
  784. cognite_toolkit/_cdf_tk/prototypes/_packages/reference/references/auth/reference.SecurityCategory.yaml +0 -1
  785. cognite_toolkit/_cdf_tk/prototypes/_packages/reference/references/data_models/reference.DataModel.yaml +0 -19
  786. cognite_toolkit/_cdf_tk/prototypes/_packages/reference/references/data_models/reference.View.yaml +0 -8
  787. cognite_toolkit/_cdf_tk/prototypes/_packages/reference/references/data_sets/reference.DataSet.yaml +0 -8
  788. cognite_toolkit/_cdf_tk/prototypes/_packages/reference/references/extraction_pipelines/reference.ExtractionPipeline.yaml +0 -27
  789. cognite_toolkit/_cdf_tk/prototypes/_packages/reference/references/extraction_pipelines/reference.ExtractionPipelineConfig.yaml +0 -3
  790. cognite_toolkit/_cdf_tk/prototypes/_packages/reference/references/functions/reference.Function.yaml +0 -22
  791. cognite_toolkit/_cdf_tk/prototypes/_packages/reference/references/functions/reference.FunctionSchedule.yaml +0 -6
  792. cognite_toolkit/_cdf_tk/prototypes/_packages/reference/references/labels/reference.LabelDefinition.yaml +0 -4
  793. cognite_toolkit/_cdf_tk/prototypes/_packages/reference/references/timeseries/reference.DataPointSubscription.yaml +0 -7
  794. cognite_toolkit/_cdf_tk/prototypes/_packages/reference/references/transformations/reference.Transformation.yaml +0 -51
  795. cognite_toolkit/_cdf_tk/prototypes/_packages/reference/references/transformations/reference.TransformationNotification.yaml +0 -3
  796. cognite_toolkit/_cdf_tk/prototypes/_packages/reference/references/transformations/reference.TransformationSchedule.yaml +0 -4
  797. cognite_toolkit/_cdf_tk/prototypes/_packages/reference/references/workflows/reference.WorkflowUpsert.yaml +0 -2
  798. cognite_toolkit/_cdf_tk/prototypes/_packages/reference/references/workflows/reference.WorkflowVersionUpsert.yaml +0 -10
  799. cognite_toolkit/_cdf_tk/prototypes/_packages/update_references.py +0 -339
  800. cognite_toolkit/_cdf_tk/prototypes/commands/__init__.py +0 -0
  801. cognite_toolkit/_cdf_tk/prototypes/import_app.py +0 -41
  802. cognite_toolkit/_cdf_tk/utils/table_writers.py +0 -433
  803. cognite_toolkit/_cdf_tk/utils/tarjan.py +0 -46
  804. cognite_toolkit-0.5.44.dist-info/RECORD +0 -695
  805. cognite_toolkit-0.5.44.dist-info/WHEEL +0 -4
  806. cognite_toolkit-0.5.44.dist-info/licenses/LICENSE +0 -18
  807. /cognite_toolkit/{_builtin_modules → _cdf_tk/commands/build_v2}/__init__.py +0 -0
@@ -1,197 +1,559 @@
1
- from __future__ import annotations
2
-
3
1
  import uuid
4
- from collections.abc import Hashable
5
- from graphlib import CycleError, TopologicalSorter
6
- from typing import cast
2
+ from abc import ABC, abstractmethod
3
+ from collections.abc import Callable, Hashable, Iterable, Sequence
4
+ from dataclasses import dataclass
5
+ from functools import partial
6
+ from typing import Literal, cast
7
7
 
8
8
  import questionary
9
- from cognite.client.data_classes import AggregateResultItem, DataSetUpdate, filters
10
- from cognite.client.data_classes.data_modeling import NodeId
9
+ from cognite.client.data_classes import DataSetUpdate
10
+ from cognite.client.data_classes.data_modeling import Edge, NodeId
11
+ from cognite.client.data_classes.data_modeling.statistics import SpaceStatistics
11
12
  from cognite.client.exceptions import CogniteAPIError
13
+ from cognite.client.utils._identifier import InstanceId
12
14
  from rich import print
13
15
  from rich.console import Console
14
16
  from rich.panel import Panel
15
- from rich.status import Status
16
17
 
17
18
  from cognite_toolkit._cdf_tk.client import ToolkitClient
19
+ from cognite_toolkit._cdf_tk.cruds import (
20
+ AssetCRUD,
21
+ ContainerCRUD,
22
+ DataModelCRUD,
23
+ EdgeCRUD,
24
+ EventCRUD,
25
+ ExtractionPipelineCRUD,
26
+ FileMetadataCRUD,
27
+ LabelCRUD,
28
+ NodeCRUD,
29
+ RelationshipCRUD,
30
+ ResourceCRUD,
31
+ SequenceCRUD,
32
+ SpaceCRUD,
33
+ ThreeDModelCRUD,
34
+ TimeSeriesCRUD,
35
+ TransformationCRUD,
36
+ ViewCRUD,
37
+ WorkflowCRUD,
38
+ )
18
39
  from cognite_toolkit._cdf_tk.data_classes import DeployResults, ResourceDeployResult
19
40
  from cognite_toolkit._cdf_tk.exceptions import (
20
- CDFAPIError,
41
+ AuthorizationError,
21
42
  ToolkitMissingResourceError,
22
- ToolkitRequiredValueError,
23
- ToolkitValueError,
24
43
  )
25
- from cognite_toolkit._cdf_tk.loaders import (
26
- RESOURCE_LOADER_LIST,
27
- AssetLoader,
28
- CogniteFileLoader,
29
- DataSetsLoader,
30
- FunctionLoader,
31
- GraphQLLoader,
32
- GroupAllScopedLoader,
33
- GroupLoader,
34
- GroupResourceScopedLoader,
35
- HostedExtractorDestinationLoader,
36
- LocationFilterLoader,
37
- NodeLoader,
38
- ResourceLoader,
39
- SpaceLoader,
40
- StreamlitLoader,
41
- TransformationLoader,
42
- ViewLoader,
44
+ from cognite_toolkit._cdf_tk.protocols import ResourceResponseProtocol
45
+ from cognite_toolkit._cdf_tk.storageio import InstanceIO
46
+ from cognite_toolkit._cdf_tk.storageio.selectors import InstanceSelector
47
+ from cognite_toolkit._cdf_tk.tk_warnings import (
48
+ HighSeverityWarning,
49
+ LimitedAccessWarning,
43
50
  )
44
- from cognite_toolkit._cdf_tk.tk_warnings import HighSeverityWarning, MediumSeverityWarning
45
51
  from cognite_toolkit._cdf_tk.utils import humanize_collection
52
+ from cognite_toolkit._cdf_tk.utils.aggregators import (
53
+ AssetAggregator,
54
+ EventAggregator,
55
+ FileAggregator,
56
+ LabelCountAggregator,
57
+ RelationshipAggregator,
58
+ SequenceAggregator,
59
+ TimeSeriesAggregator,
60
+ )
61
+ from cognite_toolkit._cdf_tk.utils.http_client import (
62
+ FailedRequestItems,
63
+ FailedResponseItems,
64
+ HTTPClient,
65
+ ItemsRequest,
66
+ SuccessResponseItems,
67
+ )
68
+ from cognite_toolkit._cdf_tk.utils.producer_worker import ProducerWorkerExecutor
69
+ from cognite_toolkit._cdf_tk.utils.useful_types import JsonVal
70
+ from cognite_toolkit._cdf_tk.utils.validate_access import ValidateAccess
46
71
 
47
72
  from ._base import ToolkitCommand
48
73
 
49
74
 
75
+ @dataclass
76
+ class DeleteResults:
77
+ deleted: int = 0
78
+ failed: int = 0
79
+
80
+
81
+ @dataclass
82
+ class DeleteItem:
83
+ item: JsonVal
84
+ as_id_fun: Callable[[JsonVal], Hashable]
85
+
86
+ def dump(self) -> JsonVal:
87
+ return self.item
88
+
89
+ def as_id(self) -> Hashable:
90
+ return self.as_id_fun(self.item)
91
+
92
+
93
+ @dataclass
94
+ class ToDelete(ABC):
95
+ crud: ResourceCRUD
96
+ total: int
97
+ delete_url: str
98
+
99
+ @property
100
+ def display_name(self) -> str:
101
+ return self.crud.display_name
102
+
103
+ @abstractmethod
104
+ def get_process_function(
105
+ self, client: ToolkitClient, console: Console, verbose: bool, process_results: ResourceDeployResult
106
+ ) -> Callable[[list[ResourceResponseProtocol]], list[JsonVal]]:
107
+ raise NotImplementedError()
108
+
109
+ def get_extra_fields(self) -> dict[str, JsonVal]:
110
+ return {}
111
+
112
+
113
+ @dataclass
114
+ class DataModelingToDelete(ToDelete):
115
+ def get_process_function(
116
+ self, client: ToolkitClient, console: Console, verbose: bool, process_results: ResourceDeployResult
117
+ ) -> Callable[[list[ResourceResponseProtocol]], list[JsonVal]]:
118
+ def as_id(chunk: list[ResourceResponseProtocol]) -> list[JsonVal]:
119
+ # We know that all data modeling resources implement as_id
120
+ return [item.as_id().dump(include_type=False) for item in chunk] # type: ignore[attr-defined]
121
+
122
+ return as_id
123
+
124
+
125
+ @dataclass
126
+ class EdgeToDelete(ToDelete):
127
+ def get_process_function(
128
+ self, client: ToolkitClient, console: Console, verbose: bool, process_results: ResourceDeployResult
129
+ ) -> Callable[[list[ResourceResponseProtocol]], list[JsonVal]]:
130
+ def as_id(chunk: list[ResourceResponseProtocol]) -> list[JsonVal]:
131
+ return [
132
+ {"space": item.space, "externalId": item.external_id, "instanceType": "edge"}
133
+ for item in cast(list[Edge], chunk)
134
+ ]
135
+
136
+ return as_id
137
+
138
+
139
+ @dataclass
140
+ class NodesToDelete(ToDelete):
141
+ delete_datapoints: bool
142
+ delete_file_content: bool
143
+
144
+ def get_process_function(
145
+ self, client: ToolkitClient, console: Console, verbose: bool, process_results: ResourceDeployResult
146
+ ) -> Callable[[list[ResourceResponseProtocol]], list[JsonVal]]:
147
+ def check_for_data(chunk: list[ResourceResponseProtocol]) -> list[JsonVal]:
148
+ # We know that all node resources implement as_id
149
+ node_ids = [item.as_id() for item in chunk] # type: ignore[attr-defined]
150
+ found_ids: set[InstanceId] = set()
151
+ if not self.delete_datapoints:
152
+ timeseries = client.time_series.retrieve_multiple(instance_ids=node_ids, ignore_unknown_ids=True)
153
+ found_ids |= {ts.instance_id for ts in timeseries if ts.instance_id is not None}
154
+ if not self.delete_file_content:
155
+ files = client.files.retrieve_multiple(instance_ids=node_ids, ignore_unknown_ids=True)
156
+ found_ids |= {f.instance_id for f in files if f.instance_id is not None}
157
+ if found_ids and verbose:
158
+ console.print(f"Skipping {found_ids} nodes as they have datapoints or file content")
159
+ process_results.unchanged += len(found_ids)
160
+ result: list[JsonVal] = []
161
+ for node_id in (n for n in node_ids if n not in found_ids):
162
+ dumped = node_id.dump(include_instance_type=True)
163
+ # The delete endpoint expects "instanceType" instead of "type"
164
+ dumped["instanceType"] = dumped.pop("type")
165
+ result.append(dumped)
166
+ return result
167
+
168
+ return check_for_data
169
+
170
+
171
+ @dataclass
172
+ class IdResourceToDelete(ToDelete):
173
+ def get_process_function(
174
+ self, client: ToolkitClient, console: Console, verbose: bool, process_results: ResourceDeployResult
175
+ ) -> Callable[[list[ResourceResponseProtocol]], list[JsonVal]]:
176
+ def as_id(chunk: list[ResourceResponseProtocol]) -> list[JsonVal]:
177
+ # We know that all id resources have an id attribute
178
+ return [{"id": item.id} for item in chunk] # type: ignore[attr-defined]
179
+
180
+ return as_id
181
+
182
+
183
+ @dataclass
184
+ class ExternalIdToDelete(ToDelete):
185
+ def get_process_function(
186
+ self, client: ToolkitClient, console: Console, verbose: bool, process_results: ResourceDeployResult
187
+ ) -> Callable[[list[ResourceResponseProtocol]], list[JsonVal]]:
188
+ def as_external_id(chunk: list[ResourceResponseProtocol]) -> list[JsonVal]:
189
+ # We know that all external id resources have an external_id attribute
190
+ return [{"externalId": item.external_id} for item in chunk] # type: ignore[attr-defined]
191
+
192
+ return as_external_id
193
+
194
+
195
+ @dataclass
196
+ class AssetToDelete(IdResourceToDelete):
197
+ recursive: bool
198
+
199
+ def get_extra_fields(self) -> dict[str, JsonVal]:
200
+ return {"recursive": self.recursive}
201
+
202
+
50
203
  class PurgeCommand(ToolkitCommand):
204
+ BATCH_SIZE_DM = 1000
205
+
51
206
  def space(
52
207
  self,
53
208
  client: ToolkitClient,
54
- space: str | None = None,
209
+ selected_space: str,
55
210
  include_space: bool = False,
211
+ delete_datapoints: bool = False,
212
+ delete_file_content: bool = False,
56
213
  dry_run: bool = False,
57
214
  auto_yes: bool = False,
58
215
  verbose: bool = False,
59
- ) -> None:
60
- """Purge a space and all its content"""
61
- selected_space = self._get_selected_space(space, client)
62
- if space is None:
63
- # Interactive mode
64
- include_space = questionary.confirm("Do you also want to delete the space itself?", default=False).ask()
65
- dry_run = questionary.confirm("Dry run?", default=True).ask()
216
+ ) -> DeployResults:
217
+ # Warning Messages
66
218
  if not dry_run:
67
219
  self._print_panel("space", selected_space)
68
- if not auto_yes:
69
- confirm = questionary.confirm(
70
- f"Are you really sure you want to purge the {selected_space!r} space?", default=False
71
- ).ask()
72
- if not confirm:
73
- return
74
-
75
- loaders = self._get_dependencies(
76
- SpaceLoader,
77
- exclude={
78
- GraphQLLoader,
79
- GroupResourceScopedLoader,
80
- LocationFilterLoader,
81
- TransformationLoader,
82
- CogniteFileLoader,
83
- },
84
- )
85
- is_purged = self._purge(client, loaders, selected_space, dry_run=dry_run, verbose=verbose)
86
- if include_space and is_purged:
87
- space_loader = SpaceLoader.create_loader(client)
88
- if dry_run:
89
- print(f"Would delete space {selected_space}")
90
- else:
91
- try:
92
- space_loader.delete([selected_space])
93
- print(f"Space {selected_space} deleted")
94
- except CogniteAPIError as e:
95
- self.warn(HighSeverityWarning(f"Failed to delete space {selected_space!r}: {e}"))
96
- elif include_space:
97
- self.warn(HighSeverityWarning(f"Space {selected_space!r} was not deleted due to errors during the purge"))
98
-
99
- if not dry_run and is_purged:
100
- print(f"Purge space {selected_space!r} completed.")
101
- elif not dry_run:
102
- print(f"Purge space {selected_space!r} partly completed. See warnings for details.")
103
220
 
104
- @staticmethod
105
- def _get_dependencies(
106
- loader_cls: type[ResourceLoader], exclude: set[type[ResourceLoader]] | None = None
107
- ) -> dict[type[ResourceLoader], frozenset[type[ResourceLoader]]]:
108
- return {
109
- dep_cls: dep_cls.dependencies
110
- for dep_cls in RESOURCE_LOADER_LIST
111
- if loader_cls in dep_cls.dependencies and (exclude is None or dep_cls not in exclude)
112
- }
221
+ if not dry_run and not auto_yes:
222
+ confirm = questionary.confirm(
223
+ f"Are you really sure you want to purge the {selected_space!r} space?", default=False
224
+ ).ask()
225
+ if not confirm:
226
+ return DeployResults([], "purge", dry_run=dry_run)
227
+
228
+ stats = client.data_modeling.statistics.spaces.retrieve(selected_space)
229
+ if stats is None:
230
+ raise ToolkitMissingResourceError(f"Space {selected_space!r} does not exist")
231
+
232
+ # ValidateAuth
233
+ validator = ValidateAccess(client, "purge")
234
+ if include_space or (stats.containers + stats.views + stats.data_models) > 0:
235
+ # We check for write even in dry-run mode. This is because dry-run is expected to fail
236
+ # if the user cannot perform the purge.
237
+ validator.data_model(["read", "write"], spaces={selected_space})
238
+ if (stats.nodes + stats.edges) > 0:
239
+ validator.instances(["read", "write"], spaces={selected_space})
240
+
241
+ to_delete = self._create_to_delete_list_purge_space(client, delete_datapoints, delete_file_content, stats)
242
+ if dry_run:
243
+ results = DeployResults([], "purge", dry_run=True)
244
+ for item in to_delete:
245
+ results[item.display_name] = ResourceDeployResult(item.display_name, deleted=item.total)
246
+ if include_space:
247
+ space_loader = SpaceCRUD.create_loader(client)
248
+ results[space_loader.display_name] = ResourceDeployResult(space_loader.display_name, deleted=1)
249
+ else:
250
+ results = self._delete_resources(to_delete, client, verbose, selected_space, None)
251
+ if include_space:
252
+ self._delete_space(client, selected_space, results)
253
+ print(results.counts_table(exclude_columns={"Created", "Changed", "Total"}))
254
+ return results
113
255
 
114
256
  @staticmethod
115
- def _get_selected_space(space: str | None, client: ToolkitClient) -> str:
116
- if space is None:
117
- spaces = client.data_modeling.spaces.list(limit=-1, include_global=False)
118
- selected_space = questionary.select(
119
- "Which space do you want to purge"
120
- " (including all data models, views, containers, nodes and edges within that space)?",
121
- sorted([space.space for space in spaces]),
122
- ).ask()
257
+ def _create_to_delete_list_purge_space(
258
+ client: ToolkitClient, delete_datapoints: bool, delete_file_content: bool, stats: SpaceStatistics
259
+ ) -> list[ToDelete]:
260
+ config = client.config
261
+ to_delete = [
262
+ EdgeToDelete(
263
+ EdgeCRUD.create_loader(client), stats.edges, config.create_api_url("/models/instances/delete")
264
+ ),
265
+ NodesToDelete(
266
+ NodeCRUD.create_loader(client),
267
+ stats.nodes,
268
+ config.create_api_url(
269
+ "/models/instances/delete",
270
+ ),
271
+ delete_datapoints=delete_datapoints,
272
+ delete_file_content=delete_file_content,
273
+ ),
274
+ DataModelingToDelete(
275
+ DataModelCRUD.create_loader(client),
276
+ stats.data_models,
277
+ config.create_api_url("/models/datamodels/delete"),
278
+ ),
279
+ DataModelingToDelete(
280
+ ViewCRUD.create_loader(client), stats.views, config.create_api_url("/models/views/delete")
281
+ ),
282
+ DataModelingToDelete(
283
+ ContainerCRUD.create_loader(client),
284
+ stats.containers,
285
+ config.create_api_url("/models/containers/delete"),
286
+ ),
287
+ ]
288
+ return to_delete
289
+
290
+ def _delete_space(self, client: ToolkitClient, selected_space: str, results: DeployResults) -> None:
291
+ space_loader = SpaceCRUD.create_loader(client)
292
+ try:
293
+ space_loader.delete([selected_space])
294
+ print(f"Space {selected_space} deleted")
295
+ except CogniteAPIError as e:
296
+ self.warn(HighSeverityWarning(f"Failed to delete space {selected_space!r}: {e}"))
123
297
  else:
124
- retrieved = client.data_modeling.spaces.retrieve(space)
125
- if retrieved is None:
126
- raise ToolkitMissingResourceError(f"Space {space} does not exist")
127
- selected_space = space
298
+ results[space_loader.display_name] = ResourceDeployResult(space_loader.display_name, deleted=1)
128
299
 
129
- if selected_space is None:
130
- raise ToolkitValueError("No space selected")
131
- return selected_space
300
+ def _delete_resources(
301
+ self,
302
+ to_delete: list[ToDelete],
303
+ client: ToolkitClient,
304
+ verbose: bool,
305
+ space: str | None,
306
+ data_set_external_id: str | None,
307
+ ) -> DeployResults:
308
+ results = DeployResults([], "purge", dry_run=False)
309
+ console = Console()
310
+ with HTTPClient(client.config, max_retries=10) as delete_client:
311
+ for item in to_delete:
312
+ if item.total == 0:
313
+ results[item.display_name] = ResourceDeployResult(item.display_name, deleted=0)
314
+ continue
315
+ # Two results objects since they are updated concurrently
316
+ process_results = ResourceDeployResult(item.display_name)
317
+ write_results = ResourceDeployResult(item.display_name)
318
+ iteration_count: int | None = None
319
+ if item.total > 0:
320
+ iteration_count = item.total // self.BATCH_SIZE_DM + (
321
+ 1 if item.total % self.BATCH_SIZE_DM > 0 else 0
322
+ )
323
+ executor = ProducerWorkerExecutor[list[ResourceResponseProtocol], list[JsonVal]](
324
+ download_iterable=self._iterate_batch(
325
+ item.crud, space, data_set_external_id, batch_size=self.BATCH_SIZE_DM
326
+ ),
327
+ process=item.get_process_function(client, console, verbose, process_results),
328
+ write=self._purge_batch(item, item.delete_url, delete_client, write_results),
329
+ max_queue_size=10,
330
+ iteration_count=iteration_count,
331
+ download_description=f"Downloading {item.display_name}",
332
+ process_description=f"Preparing {item.display_name} for deletion",
333
+ write_description=f"Deleting {item.display_name}",
334
+ console=console,
335
+ )
336
+ executor.run()
337
+ write_results += process_results
338
+ results[item.display_name] = write_results
339
+ if executor.error_occurred:
340
+ if verbose and executor.error_traceback:
341
+ executor.print_traceback()
342
+ self.warn(
343
+ HighSeverityWarning(f"Failed to delete all {item.display_name}. {executor.error_message}")
344
+ )
345
+ return results
346
+
347
+ @staticmethod
348
+ def _iterate_batch(
349
+ crud: ResourceCRUD, selected_space: str | None, data_set_external_id: str | None, batch_size: int
350
+ ) -> Iterable[list[ResourceResponseProtocol]]:
351
+ batch: list[ResourceResponseProtocol] = []
352
+ for resource in crud.iterate(space=selected_space, data_set_external_id=data_set_external_id):
353
+ batch.append(resource)
354
+ if len(batch) >= batch_size:
355
+ yield batch
356
+ batch = []
357
+ if batch:
358
+ yield batch
359
+
360
+ @staticmethod
361
+ def _purge_batch(
362
+ delete_item: ToDelete, delete_url: str, delete_client: HTTPClient, result: ResourceDeployResult
363
+ ) -> Callable[[list[JsonVal]], None]:
364
+ crud = delete_item.crud
365
+
366
+ def as_id(item: JsonVal) -> Hashable:
367
+ try:
368
+ return crud.get_id(item)
369
+ except KeyError:
370
+ # Fallback to internal ID
371
+ return crud.get_internal_id(item)
372
+
373
+ def process(items: list[JsonVal]) -> None:
374
+ responses = delete_client.request_with_retries(
375
+ ItemsRequest(
376
+ endpoint_url=delete_url,
377
+ method="POST",
378
+ items=[DeleteItem(item=item, as_id_fun=as_id) for item in items],
379
+ extra_body_fields=delete_item.get_extra_fields(),
380
+ )
381
+ )
382
+ for response in responses:
383
+ if isinstance(response, SuccessResponseItems):
384
+ result.deleted += len(response.ids)
385
+ else:
386
+ result.unchanged += len(items)
387
+
388
+ return process
132
389
 
133
390
  def dataset(
134
391
  self,
135
392
  client: ToolkitClient,
136
- external_id: str | None = None,
137
- include_dataset: bool = False,
393
+ selected_data_set_external_id: str,
394
+ archive_dataset: bool = False,
395
+ include_data: bool = True,
396
+ include_configurations: bool = False,
397
+ asset_recursive: bool = False,
138
398
  dry_run: bool = False,
139
399
  auto_yes: bool = False,
140
400
  verbose: bool = False,
141
- ) -> None:
142
- """Purge a dataset and all its content"""
143
- selected_dataset = self._get_selected_dataset(external_id, client)
144
- if external_id is None:
145
- # Interactive mode
146
- include_dataset = questionary.confirm(
147
- "Do you want to archive the dataset itself after the purge?", default=False
148
- ).ask()
149
- dry_run = questionary.confirm("Dry run?", default=True).ask()
401
+ ) -> DeployResults:
402
+ """Purge a dataset and all its content
403
+
404
+ Args:
405
+ client: The ToolkitClient to use
406
+ selected_data_set_external_id: The external ID of the dataset to purge
407
+ archive_dataset: Whether to archive the dataset itself after the purge
408
+ include_data: Whether to include data (assets, events, time series, files, sequences, 3D models, relationships, labels) in the purge
409
+ include_configurations: Whether to include configurations (workflows, transformations, extraction pipelines) in the purge
410
+ asset_recursive: Whether to recursively delete assets.
411
+ dry_run: Whether to perform a dry run
412
+ auto_yes: Whether to automatically confirm the purge
413
+ verbose: Whether to print verbose output
414
+
415
+ Returns:
416
+ DeployResults: The results of the purge operation
417
+
418
+ """
419
+ # Warning Messages
150
420
  if not dry_run:
151
- self._print_panel("dataset", selected_dataset)
152
- if not auto_yes:
153
- confirm = questionary.confirm(
154
- f"Are you really sure you want to purge the {selected_dataset!r} dataset?", default=False
155
- ).ask()
156
- if not confirm:
157
- return
158
-
159
- loaders = self._get_dependencies(
160
- DataSetsLoader,
161
- exclude={
162
- GroupLoader,
163
- GroupResourceScopedLoader,
164
- GroupAllScopedLoader,
165
- StreamlitLoader,
166
- HostedExtractorDestinationLoader,
167
- FunctionLoader,
168
- LocationFilterLoader,
169
- },
421
+ self._print_panel("dataSet", selected_data_set_external_id)
422
+ if not dry_run and not auto_yes:
423
+ confirm = questionary.confirm(
424
+ f"Are you really sure you want to purge the {selected_data_set_external_id!r} dataSet?", default=False
425
+ ).ask()
426
+ if not confirm:
427
+ return DeployResults([], "purge", dry_run=dry_run)
428
+
429
+ # Validate Auth
430
+ validator = ValidateAccess(client, "purge")
431
+ data_set_id = client.lookup.data_sets.id(selected_data_set_external_id)
432
+ if data_set_id is None:
433
+ raise ToolkitMissingResourceError(f"DataSet {selected_data_set_external_id!r} does not exist")
434
+ action = cast(Sequence[Literal["read", "write"]], ["read"] if dry_run else ["read", "write"])
435
+ if include_data:
436
+ # Check asset, events, time series, files, and sequences access, relationships, labels, 3D access.
437
+ validator.dataset_data(action, dataset_ids={data_set_id})
438
+ if include_configurations:
439
+ # Check workflow, transformations, extraction pipeline access
440
+ validator.dataset_configurations(action, dataset_ids={data_set_id})
441
+
442
+ to_delete: list[ToDelete] = self._create_to_delete_list_purge_dataset(
443
+ client,
444
+ include_data,
445
+ include_configurations,
446
+ selected_data_set_external_id,
447
+ asset_recursive,
170
448
  )
171
- is_purged = self._purge(client, loaders, selected_data_set=selected_dataset, dry_run=dry_run, verbose=verbose)
172
- if include_dataset and is_purged:
173
- if dry_run:
174
- print(f"Would have archived {selected_dataset}")
175
- else:
176
- archived = (
177
- DataSetUpdate(external_id=selected_dataset)
178
- .external_id.set(str(uuid.uuid4()))
179
- .metadata.add({"archived": "true"})
180
- .write_protected.set(True)
181
- )
182
- client.data_sets.update(archived)
183
- print(f"DataSet {selected_dataset} archived")
184
- elif include_dataset:
185
- self.warn(
186
- HighSeverityWarning(f"DataSet {selected_dataset} was not archived due to errors during the purge")
187
- )
449
+ if dry_run:
450
+ results = DeployResults([], "purge", dry_run=True)
451
+ for item in to_delete:
452
+ results[item.display_name] = ResourceDeployResult(item.display_name, deleted=item.total)
453
+ else:
454
+ results = self._delete_resources(to_delete, client, verbose, None, selected_data_set_external_id)
455
+ print(results.counts_table(exclude_columns={"Created", "Changed", "Total"}))
456
+ if archive_dataset and not dry_run:
457
+ self._archive_dataset(client, selected_data_set_external_id)
458
+ return results
188
459
 
189
- if not dry_run and is_purged:
190
- print(f"Purged dataset {selected_dataset!r} completed")
191
- elif not dry_run:
192
- print(f"Purged dataset {selected_dataset!r} partly completed. See warnings for details.")
460
+ @staticmethod
461
+ def _archive_dataset(client: ToolkitClient, data_set: str) -> None:
462
+ archived = (
463
+ DataSetUpdate(external_id=data_set)
464
+ .external_id.set(str(uuid.uuid4()))
465
+ .metadata.add({"archived": "true"})
466
+ .write_protected.set(True)
467
+ )
468
+ client.data_sets.update(archived)
469
+ print(f"DataSet {data_set} archived")
193
470
 
194
- def _print_panel(self, resource_type: str, resource: str) -> None:
471
+ @staticmethod
472
+ def _create_to_delete_list_purge_dataset(
473
+ client: ToolkitClient,
474
+ include_data: bool,
475
+ include_configurations: bool,
476
+ data_set_external_id: str,
477
+ asset_recursive: bool,
478
+ ) -> list[ToDelete]:
479
+ config = client.config
480
+ to_delete: list[ToDelete] = []
481
+
482
+ if include_data:
483
+ three_d_crud = ThreeDModelCRUD.create_loader(client)
484
+ to_delete.extend(
485
+ [
486
+ ExternalIdToDelete(
487
+ RelationshipCRUD.create_loader(client),
488
+ RelationshipAggregator(client).count(data_set_external_id=data_set_external_id),
489
+ config.create_api_url("/relationships/delete"),
490
+ ),
491
+ IdResourceToDelete(
492
+ EventCRUD.create_loader(client),
493
+ EventAggregator(client).count(data_set_external_id=data_set_external_id),
494
+ config.create_api_url("/events/delete"),
495
+ ),
496
+ IdResourceToDelete(
497
+ FileMetadataCRUD.create_loader(client),
498
+ FileAggregator(client).count(data_set_external_id=data_set_external_id),
499
+ config.create_api_url("/files/delete"),
500
+ ),
501
+ IdResourceToDelete(
502
+ TimeSeriesCRUD.create_loader(client),
503
+ TimeSeriesAggregator(client).count(data_set_external_id=data_set_external_id),
504
+ config.create_api_url("/timeseries/delete"),
505
+ ),
506
+ IdResourceToDelete(
507
+ SequenceCRUD.create_loader(client),
508
+ SequenceAggregator(client).count(data_set_external_id=data_set_external_id),
509
+ config.create_api_url("/sequences/delete"),
510
+ ),
511
+ IdResourceToDelete(
512
+ three_d_crud,
513
+ sum(1 for _ in three_d_crud.iterate(data_set_external_id=data_set_external_id)),
514
+ config.create_api_url("/3d/models/delete"),
515
+ ),
516
+ AssetToDelete(
517
+ AssetCRUD.create_loader(client),
518
+ AssetAggregator(client).count(data_set_external_id=data_set_external_id),
519
+ config.create_api_url("/assets/delete"),
520
+ recursive=asset_recursive,
521
+ ),
522
+ ExternalIdToDelete(
523
+ LabelCRUD.create_loader(client),
524
+ LabelCountAggregator(client).count(data_set_external_id=data_set_external_id),
525
+ config.create_api_url("/labels/delete"),
526
+ ),
527
+ ]
528
+ )
529
+ if include_configurations:
530
+ transformation_crud = TransformationCRUD.create_loader(client)
531
+ workflow_crud = WorkflowCRUD.create_loader(client)
532
+ extraction_pipeline_crud = ExtractionPipelineCRUD.create_loader(client)
533
+
534
+ to_delete.extend(
535
+ [
536
+ IdResourceToDelete(
537
+ transformation_crud,
538
+ sum(1 for _ in transformation_crud.iterate(data_set_external_id=data_set_external_id)),
539
+ config.create_api_url("/transformations/delete"),
540
+ ),
541
+ ExternalIdToDelete(
542
+ workflow_crud,
543
+ sum(1 for _ in workflow_crud.iterate(data_set_external_id=data_set_external_id)),
544
+ config.create_api_url("/workflows/delete"),
545
+ ),
546
+ IdResourceToDelete(
547
+ extraction_pipeline_crud,
548
+ sum(1 for _ in extraction_pipeline_crud.iterate(data_set_external_id=data_set_external_id)),
549
+ config.create_api_url("/extpipes/delete"),
550
+ ),
551
+ ]
552
+ )
553
+ return to_delete
554
+
555
+ @staticmethod
556
+ def _print_panel(resource_type: str, resource: str) -> None:
195
557
  print(
196
558
  Panel(
197
559
  f"[red]WARNING:[/red] This operation [bold]cannot be undone[/bold]! "
@@ -204,383 +566,202 @@ class PurgeCommand(ToolkitCommand):
204
566
  )
205
567
  )
206
568
 
207
- @staticmethod
208
- def _get_selected_dataset(external_id: str | None, client: ToolkitClient) -> str:
209
- if external_id is None:
210
- datasets = client.data_sets.list(limit=-1)
211
- selected_dataset: str = questionary.select(
212
- "Which space are you going to purge (delete all resources in dataset)?",
213
- sorted([dataset.external_id for dataset in datasets if dataset.external_id]),
214
- ).ask()
215
- else:
216
- retrieved = client.data_sets.retrieve(external_id=external_id)
217
- if retrieved is None:
218
- raise ToolkitMissingResourceError(f"DataSet {external_id!r} does not exist")
219
- selected_dataset = external_id
220
-
221
- if selected_dataset is None:
222
- raise ToolkitValueError("No space selected")
223
- return selected_dataset
224
-
225
- def _purge(
569
+ def instances(
226
570
  self,
227
571
  client: ToolkitClient,
228
- loaders: dict[type[ResourceLoader], frozenset[type[ResourceLoader]]],
229
- selected_space: str | None = None,
230
- selected_data_set: str | None = None,
572
+ selector: InstanceSelector,
231
573
  dry_run: bool = False,
574
+ auto_yes: bool = False,
575
+ unlink: bool = True,
232
576
  verbose: bool = False,
233
- batch_size: int = 1000,
234
- ) -> bool:
235
- is_purged = True
236
- results = DeployResults([], "purge", dry_run=dry_run)
237
- loader_cls: type[ResourceLoader]
238
- has_purged_views = False
239
- with Console().status("...", spinner="aesthetic", speed=0.4) as status:
240
- for loader_cls in reversed(list(TopologicalSorter(loaders).static_order())):
241
- if loader_cls not in loaders:
242
- # Dependency that is included
243
- continue
244
- loader = loader_cls.create_loader(client, console=status.console)
245
- status_prefix = "Would have deleted" if dry_run else "Deleted"
246
- if isinstance(loader, ViewLoader) and not dry_run:
247
- status_prefix = "Expected deleted" # Views are not always deleted immediately
248
- has_purged_views = True
249
-
250
- if not dry_run and isinstance(loader, NodeLoader):
251
- # Special handling of nodes as node type must be deleted after regular nodes
252
- # In dry-run mode, we are not deleting the nodes, so we can skip this.
253
- warnings_before = len(self.warning_list)
254
- deleted_nodes = self._purge_nodes(loader, status, selected_space, verbose)
255
- results[loader.display_name] = ResourceDeployResult(
256
- name=loader.display_name,
257
- deleted=deleted_nodes,
258
- total=deleted_nodes,
259
- )
260
- if len(self.warning_list) > warnings_before and any(
261
- isinstance(warn, HighSeverityWarning) for warn in self.warning_list[warnings_before:]
262
- ):
263
- is_purged = False
264
- continue
265
- elif not dry_run and isinstance(loader, AssetLoader):
266
- # Special handling of assets as we must ensure all children are deleted before the parent.
267
- # In dry-run mode, we are not deleting the assets, so we can skip this.
268
- deleted_assets = self._purge_assets(loader, status, selected_data_set)
269
- results[loader.display_name] = ResourceDeployResult(
270
- loader.display_name, deleted=deleted_assets, total=deleted_assets
271
- )
272
- continue
577
+ ) -> DeleteResults:
578
+ """Purge instances"""
579
+ io = InstanceIO(client)
580
+ console = Console()
581
+ validator = ValidateAccess(client, default_operation="purge")
582
+ self.validate_instance_access(validator, selector.get_instance_spaces())
583
+ if unlink:
584
+ self.validate_timeseries_access(validator)
585
+ self.validate_file_access(validator)
586
+
587
+ total = io.count(selector)
588
+ if total is None or total == 0:
589
+ print("No instances found.")
590
+ return DeleteResults()
591
+ if not dry_run:
592
+ self._print_panel("instances", str(selector))
593
+ if not auto_yes:
594
+ confirm = questionary.confirm(
595
+ f"Are you really sure you want to purge all {total:,} instances in {selector!s}?",
596
+ default=False,
597
+ ).ask()
598
+ if not confirm:
599
+ return DeleteResults()
600
+
601
+ process: Callable[[list[InstanceId]], list[dict[str, JsonVal]]] = self._prepare
602
+ if unlink:
603
+ process = partial(self._unlink_prepare, client=client, dry_run=dry_run, console=console, verbose=verbose)
604
+
605
+ iteration_count = int(total // io.CHUNK_SIZE + (1 if total % io.CHUNK_SIZE > 0 else 0))
606
+ with HTTPClient(config=client.config) as delete_client:
607
+ process_str = "Would be unlinking" if dry_run else "Unlinking"
608
+ write_str = "Would be deleting" if dry_run else "Deleting"
609
+ results = DeleteResults()
610
+ executor = ProducerWorkerExecutor[list[InstanceId], list[dict[str, JsonVal]]](
611
+ download_iterable=io.download_ids(selector),
612
+ process=process,
613
+ write=partial(self._delete_instance_ids, dry_run=dry_run, delete_client=delete_client, results=results),
614
+ iteration_count=iteration_count,
615
+ max_queue_size=10,
616
+ download_description=f"Retrieving instances from {selector!s}",
617
+ process_description=f"{process_str} instances from files/timeseries" if unlink else "",
618
+ write_description=f"{write_str} instances",
619
+ console=console,
620
+ )
273
621
 
274
- # Child loaders are, for example, WorkflowTriggerLoader, WorkflowVersionLoader for WorkflowLoader
275
- # These must delete all resources that are connected to the resource that the loader is deleting
276
- # Exclude loaders that we are already iterating over
277
- child_loader_classes = self._get_dependencies(loader_cls, exclude=set(loaders))
278
- child_loaders = [
279
- child_loader.create_loader(client)
280
- for child_loader in reversed(list(TopologicalSorter(child_loader_classes).static_order()))
281
- # Necessary as the topological sort includes dependencies that are not in the loaders
282
- if child_loader in child_loader_classes
283
- ]
284
- count = 0
285
- status.update(f"{status_prefix} {count:,} {loader.display_name}...")
286
- batch_ids: list[Hashable] = []
287
- for resource in loader.iterate(data_set_external_id=selected_data_set, space=selected_space):
288
- try:
289
- batch_ids.append(loader.get_id(resource))
290
- except (ToolkitRequiredValueError, KeyError) as e:
291
- try:
292
- batch_ids.append(loader.get_internal_id(resource))
293
- except (AttributeError, NotImplementedError):
294
- self.warn(
295
- HighSeverityWarning(
296
- f"Cannot delete {type(resource).__name__}. Failed to obtain ID: {e}"
297
- ),
298
- console=status.console,
299
- )
300
- is_purged = False
301
- continue
302
-
303
- if len(batch_ids) >= batch_size:
304
- child_deletion = self._delete_children(
305
- batch_ids, child_loaders, dry_run, status.console, verbose
306
- )
307
- batch_delete, batch_size = self._delete_batch(
308
- batch_ids, dry_run, loader, batch_size, status.console, verbose
309
- )
310
- count += batch_delete
311
- status.update(f"{status_prefix} {count:,} {loader.display_name}...")
312
- batch_ids = []
313
- # The DeployResults is overloaded such that the below accumulates the counts
314
- for name, child_count in child_deletion.items():
315
- results[name] = ResourceDeployResult(name, deleted=child_count, total=child_count)
316
-
317
- if batch_ids:
318
- child_deletion = self._delete_children(batch_ids, child_loaders, dry_run, status.console, verbose)
319
- batch_delete, batch_size = self._delete_batch(
320
- batch_ids, dry_run, loader, batch_size, status.console, verbose
321
- )
322
- count += batch_delete
323
- status.update(f"{status_prefix} {count:,} {loader.display_name}...")
324
- for name, child_count in child_deletion.items():
325
- results[name] = ResourceDeployResult(name, deleted=child_count, total=child_count)
326
- if count > 0:
327
- status.console.print(f"{status_prefix} {count:,} {loader.display_name}.")
328
- results[loader.display_name] = ResourceDeployResult(
329
- name=loader.display_name,
330
- deleted=count,
331
- total=count,
332
- )
333
- print(results.counts_table(exclude_columns={"Created", "Changed", "Untouched", "Total"}))
334
- if has_purged_views:
335
- print("You might need to run the purge command multiple times to delete all views.")
336
- return is_purged
622
+ executor.run()
623
+ executor.raise_on_error()
337
624
 
338
- def _delete_batch(
339
- self,
340
- batch_ids: list[Hashable],
341
- dry_run: bool,
342
- loader: ResourceLoader,
343
- batch_size: int,
344
- console: Console,
345
- verbose: bool,
346
- ) -> tuple[int, int]:
347
- if dry_run:
348
- deleted = len(batch_ids)
625
+ prefix = "Would have purged" if dry_run else "Purged"
626
+ if results.failed == 0:
627
+ console.print(f"{prefix} {results.deleted:,} instances in {selector!s}")
349
628
  else:
350
- try:
351
- deleted = loader.delete(batch_ids)
352
- except CogniteAPIError as delete_error:
353
- if (
354
- delete_error.code == 408
355
- and "timed out" in delete_error.message.casefold()
356
- and batch_size > 1
357
- and (len(batch_ids) > 1)
358
- ):
359
- self.warn(
360
- MediumSeverityWarning(
361
- f"Timed out deleting {loader.display_name}. Trying again with a smaller batch size."
362
- ),
363
- include_timestamp=True,
364
- console=console,
365
- )
366
- new_batch_size = len(batch_ids) // 2
367
- first = batch_ids[:new_batch_size]
368
- second = batch_ids[new_batch_size:]
369
- first_deleted, first_batch_size = self._delete_batch(
370
- first, dry_run, loader, new_batch_size, console, verbose
371
- )
372
- second_deleted, second_batch_size = self._delete_batch(
373
- second, dry_run, loader, new_batch_size, console, verbose
374
- )
375
- return first_deleted + second_deleted, min(first_batch_size, second_batch_size)
376
- else:
377
- raise delete_error
629
+ console.print(
630
+ f"{prefix} {results.deleted:,} instances in {selector!s}, but failed to purge {results.failed:,} instances"
631
+ )
632
+ return results
633
+
634
+ def validate_instance_access(self, validator: ValidateAccess, instance_spaces: list[str] | None) -> None:
635
+ space_ids = validator.instances(
636
+ ["read", "write"], spaces=set(instance_spaces) if instance_spaces else None, operation="purge"
637
+ )
638
+ if space_ids is None:
639
+ # Full access
640
+ return
641
+ self.warn(
642
+ LimitedAccessWarning(
643
+ f"You can only purge instances in the following instance spaces: {humanize_collection(space_ids)}."
644
+ )
645
+ )
646
+ return
378
647
 
379
- if verbose:
380
- prefix = "Would delete" if dry_run else "Finished purging"
381
- console.print(f"{prefix} {deleted:,} {loader.display_name}")
382
- return deleted, batch_size
648
+ def validate_model_access(self, validator: ValidateAccess, view: list[str] | None) -> None:
649
+ space = view[0] if isinstance(view, list) and view and isinstance(view[0], str) else None
650
+ if space_ids := validator.data_model(["read"], spaces={space} if space else None):
651
+ self.warn(
652
+ LimitedAccessWarning(
653
+ f"You can only select views in the {len(space_ids)} spaces you have access to: {humanize_collection(space_ids)}."
654
+ )
655
+ )
383
656
 
384
- @staticmethod
385
- def _delete_children(
386
- parent_ids: list[Hashable], child_loaders: list[ResourceLoader], dry_run: bool, console: Console, verbose: bool
387
- ) -> dict[str, int]:
388
- child_deletion: dict[str, int] = {}
389
- for child_loader in child_loaders:
390
- child_ids = set()
391
- for child in child_loader.iterate(parent_ids=parent_ids):
392
- child_ids.add(child_loader.get_id(child))
393
- count = 0
394
- if child_ids:
395
- if dry_run:
396
- count = len(child_ids)
397
- else:
398
- count = child_loader.delete(list(child_ids))
657
+ def validate_timeseries_access(self, validator: ValidateAccess) -> None:
658
+ try:
659
+ ids_by_scope = validator.timeseries(["read", "write"], operation="unlink")
660
+ except AuthorizationError as e:
661
+ self.warn(HighSeverityWarning(f"You cannot unlink time series. You need read and write access: {e!s}"))
662
+ return
663
+ if ids_by_scope is None:
664
+ return
665
+ scope_str = humanize_collection(
666
+ [f"{scope_name} ({humanize_collection(ids)})" for scope_name, ids in ids_by_scope.items()],
667
+ bind_word="and",
668
+ )
669
+ self.warn(LimitedAccessWarning(f"You can only unlink time series in the following scopes: {scope_str}."))
399
670
 
400
- if verbose:
401
- prefix = "Would delete" if dry_run else "Deleted"
402
- console.print(f"{prefix} {count:,} {child_loader.display_name}")
403
- child_deletion[child_loader.display_name] = count
404
- return child_deletion
671
+ def validate_file_access(self, validator: ValidateAccess) -> None:
672
+ try:
673
+ ids_by_scope = validator.files(["read", "write"], operation="unlink")
674
+ except AuthorizationError as e:
675
+ self.warn(HighSeverityWarning(f"You cannot unlink files. You need read and write access: {e!s}"))
676
+ return
677
+ if ids_by_scope is None:
678
+ return
679
+ scope_str = humanize_collection(
680
+ [f"{scope_name} ({humanize_collection(ids)})" for scope_name, ids in ids_by_scope.items()],
681
+ bind_word="and",
682
+ )
683
+ self.warn(LimitedAccessWarning(f"You can only unlink files in the following scopes: {scope_str}."))
405
684
 
406
- def _purge_nodes(
685
+ @staticmethod
686
+ def _prepare(instance_ids: list[InstanceId]) -> list[dict[str, JsonVal]]:
687
+ output: list[dict[str, JsonVal]] = []
688
+ for instance_id in instance_ids:
689
+ dumped = instance_id.dump(include_instance_type=True)
690
+ # The PySDK uses 'type' instead of 'instanceType' which is required by the delete endpoint
691
+ dumped["instanceType"] = dumped.pop("type")
692
+ # MyPy does not understand that InstanceId.dump() returns dict[str, JsonVal]
693
+ output.append(dumped) # type: ignore[arg-type]
694
+
695
+ return output
696
+
697
+ def _unlink_prepare(
407
698
  self,
408
- loader: NodeLoader,
409
- status: Status,
410
- selected_space: str | None = None,
699
+ instance_ids: list[InstanceId],
700
+ client: ToolkitClient,
701
+ dry_run: bool,
702
+ console: Console,
411
703
  verbose: bool = False,
412
- batch_size: int = 1000,
413
- ) -> int:
414
- """Special handling of nodes as we must ensure all node types are deleted last."""
415
- # First find all Node Types
416
- node_types: dict[NodeId, set[NodeId]] = {}
417
- total_count = 0
418
- for node in loader.iterate(space=selected_space):
419
- if node.type and (selected_space is None or node.space == selected_space):
420
- node_id = NodeId(node.type.space, node.type.external_id)
421
- node_types[node_id] = set()
422
- total_count += 1
423
- status.update(f"Looking up node.type {total_count:,}...")
424
-
425
- count = 0
426
- batch_ids: list[NodeId] = []
427
- for node in loader.iterate(space=selected_space):
428
- node_id = node.as_id()
429
- if node_id in node_types:
430
- # Skip if it is a node type
431
- continue
432
- batch_ids.append(node_id)
433
- if len(batch_ids) >= batch_size:
434
- deleted, batch_size = self._delete_node_batch(batch_ids, loader, batch_size, status.console, verbose)
435
- count += deleted
436
- status.update(f"Deleted {count:,}/{total_count} {loader.display_name}...")
437
- batch_ids = []
438
-
439
- if batch_ids:
440
- deleted, batch_size = self._delete_node_batch(batch_ids, loader, batch_size, status.console, verbose)
441
- count += deleted
442
-
443
- # Finally delete all node types, first do a lookup and topological sort to ensure deleting in the right order.
444
- # Note this is an edge case, and is a result of some strange usage. But we need to handle it.
445
- for node_type in loader.retrieve(list(node_types.keys())):
446
- if node_type.type:
447
- required = NodeId(node_type.type.space, node_type.type.external_id)
448
- if required in node_types:
449
- node_types[required].add(node_type.as_id())
704
+ ) -> list[dict[str, JsonVal]]:
705
+ self._unlink_timeseries(instance_ids, client, dry_run, console, verbose)
706
+ self._unlink_files(instance_ids, client, dry_run, console, verbose)
707
+ return self._prepare(instance_ids)
450
708
 
451
- try:
452
- node_type_ids = list(TopologicalSorter(node_types).static_order())
453
- except CycleError as e:
454
- self.warn(
455
- HighSeverityWarning(f"Failed to delete node-types: Cycle detected. {e.args}"), console=status.console
709
+ @staticmethod
710
+ def _delete_instance_ids(
711
+ items: list[dict[str, JsonVal]], dry_run: bool, delete_client: HTTPClient, results: DeleteResults
712
+ ) -> None:
713
+ if dry_run:
714
+ results.deleted += len(items)
715
+ return
716
+
717
+ responses = delete_client.request_with_retries(
718
+ ItemsRequest(
719
+ delete_client.config.create_api_url("/models/instances/delete"),
720
+ method="POST",
721
+ # MyPy does not understand that InstanceId.load handles dict[str, JsonVal]
722
+ items=[DeleteItem(item=item, as_id_fun=InstanceId.load) for item in items], # type: ignore[arg-type]
456
723
  )
457
- return count
458
-
459
- deleted, batch_size = self._delete_node_batch(node_type_ids, loader, batch_size, status.console, verbose)
460
- count += deleted
461
- if count > 0:
462
- status.console.print(f"Finished purging {loader.display_name}.")
463
- return count
464
-
465
- def _delete_node_batch(
466
- self, batch_ids: list[NodeId], loader: NodeLoader, batch_size: int, console: Console, verbose: bool
467
- ) -> tuple[int, int]:
468
- try:
469
- deleted = loader.delete(batch_ids)
470
- except CogniteAPIError as delete_error:
471
- if (
472
- delete_error.code == 400
473
- and "Attempted to delete a node which is used as a type" in delete_error.message
474
- ):
475
- # Fallback to delete one by one
476
- deleted = 0
477
- for node_id in batch_ids:
478
- try:
479
- loader.delete([node_id])
480
- deleted += 1
481
- except CogniteAPIError:
482
- is_type = filters.Equals(["node", "type"], node_id.dump(include_instance_type=False))
483
- instance_spaces = {node.space for node in loader.client.data_modeling.instances(filter=is_type)}
484
- if instance_spaces:
485
- suffix = (
486
- "It is used as a node type in the following spaces, "
487
- f"which must be purged first: {humanize_collection(instance_spaces)}"
488
- )
489
- else:
490
- suffix = "It is used as a node type, but failed to find the spaces where it is used."
491
- self.warn(HighSeverityWarning(f"Failed to delete {node_id!r}. {suffix}"), console=console)
492
- elif (
493
- delete_error.code == 408
494
- and "timed out" in delete_error.message.casefold()
495
- and batch_size > 1
496
- and (len(batch_ids) > 1)
497
- ):
498
- self.warn(
499
- MediumSeverityWarning(
500
- f"Timed out deleting {loader.display_name}. Trying again with a smaller batch size."
501
- ),
502
- include_timestamp=True,
503
- console=console,
504
- )
505
- new_batch_size = len(batch_ids) // 2
506
- first = batch_ids[:new_batch_size]
507
- second = batch_ids[new_batch_size:]
508
- first_deleted, first_batch_size = self._delete_node_batch(
509
- first, loader, new_batch_size, console, verbose
510
- )
511
- second_deleted, second_batch_size = self._delete_node_batch(
512
- second, loader, new_batch_size, console, verbose
513
- )
514
- return first_deleted + second_deleted, min(first_batch_size, second_batch_size)
724
+ )
725
+ for response in responses:
726
+ if isinstance(response, SuccessResponseItems):
727
+ results.deleted += len(response.ids)
728
+ elif isinstance(response, FailedResponseItems | FailedRequestItems):
729
+ results.failed += len(response.ids)
515
730
  else:
516
- raise delete_error
731
+ results.failed += len(items)
517
732
 
518
- if verbose:
519
- console.print(f"Deleted {deleted:,} {loader.display_name}")
520
- return deleted, batch_size
733
+ @staticmethod
734
+ def _unlink_timeseries(
735
+ instances: list[InstanceId], client: ToolkitClient, dry_run: bool, console: Console, verbose: bool
736
+ ) -> list[InstanceId]:
737
+ node_ids = [instance for instance in instances if isinstance(instance, NodeId)]
738
+ if node_ids:
739
+ timeseries = client.time_series.retrieve_multiple(instance_ids=node_ids, ignore_unknown_ids=True)
740
+ if not dry_run and timeseries:
741
+ migrated_timeseries_ids = [ts.id for ts in timeseries if ts.instance_id and ts.pending_instance_id] # type: ignore[attr-defined]
742
+ client.time_series.unlink_instance_ids(id=migrated_timeseries_ids)
743
+ if verbose:
744
+ console.print(f"Unlinked {len(migrated_timeseries_ids)} timeseries from datapoints.")
745
+ elif verbose and timeseries:
746
+ console.print(f"Would have unlinked {len(timeseries)} timeseries from datapoints.")
747
+ return instances
521
748
 
522
- def _purge_assets(
523
- self,
524
- loader: AssetLoader,
525
- status: Status,
526
- selected_data_set: str | None = None,
527
- batch_size: int = 1000,
528
- ) -> int:
529
- # Using sets to avoid duplicates
530
- children_ids: set[int] = set()
531
- parent_ids: set[int] = set()
532
- is_first = True
533
- last_failed = False
534
- count = level = depth = last_parent_count = 0
535
- total_asset_count: int | None = None
536
- # Iterate through the asset hierarchy once per depth level. This is to delete all children before the parent.
537
- while is_first or level < depth:
538
- for asset in loader.iterate(data_set_external_id=selected_data_set):
539
- aggregates = cast(AggregateResultItem, asset.aggregates)
540
- if is_first and aggregates.depth is not None:
541
- depth = max(depth, aggregates.depth)
542
-
543
- if aggregates.child_count == 0:
544
- children_ids.add(asset.id)
545
- else:
546
- parent_ids.add(asset.id)
547
-
548
- if len(children_ids) >= batch_size:
549
- count += loader.delete(list(children_ids))
550
- if total_asset_count:
551
- status.update(f"Deleted {count:,}/{total_asset_count:,} {loader.display_name}...")
552
- else:
553
- status.update(f"Deleted {count:,} {loader.display_name}...")
554
- children_ids = set()
555
-
556
- if is_first:
557
- total_asset_count = count + len(parent_ids) + len(children_ids)
558
- if children_ids:
559
- count += loader.delete(list(children_ids))
560
- status.update(f"Deleted {count:,}/{total_asset_count:,} {loader.display_name}...")
561
- children_ids = set()
562
-
563
- if len(parent_ids) == last_parent_count and last_failed:
564
- try:
565
- # Just try to delete them all at once
566
- count += loader.delete(list(parent_ids))
567
- except CogniteAPIError as e:
568
- raise CDFAPIError(
569
- f"Failed to delete {len(parent_ids)} assets. This could be due to a parent-child cycle or an "
570
- "eventual consistency issue. Wait a few seconds and try again. An alternative is to use the "
571
- "Python-SDK to delete the asset hierarchy "
572
- "`client.assets.delete(external_id='my_root_asset', recursive=True)`"
573
- ) from e
574
- else:
575
- status.update(f"Deleted {count:,}/{total_asset_count:,} {loader.display_name}...")
576
- break
577
- elif len(parent_ids) == last_parent_count:
578
- last_failed = True
579
- else:
580
- last_failed = False
581
- level += 1
582
- last_parent_count = len(parent_ids)
583
- parent_ids.clear()
584
- is_first = False
585
- status.console.print(f"Finished purging {loader.display_name}.")
586
- return count
749
+ @staticmethod
750
+ def _unlink_files(
751
+ instances: list[InstanceId], client: ToolkitClient, dry_run: bool, console: Console, verbose: bool
752
+ ) -> list[InstanceId]:
753
+ file_ids = [instance for instance in instances if isinstance(instance, NodeId)]
754
+ if file_ids:
755
+ files = client.files.retrieve_multiple(instance_ids=file_ids, ignore_unknown_ids=True)
756
+ if not dry_run and files:
757
+ migrated_file_ids = [
758
+ file.id
759
+ for file in files
760
+ if file.instance_id and file.pending_instance_id and file.id is not None # type: ignore[attr-defined]
761
+ ]
762
+ client.files.unlink_instance_ids(id=migrated_file_ids)
763
+ if verbose:
764
+ console.print(f"Unlinked {len(migrated_file_ids)} files from nodes.")
765
+ elif verbose and files:
766
+ console.print(f"Would have unlinked {len(files)} files from their blob content.")
767
+ return instances