simstack 2.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (354) hide show
  1. simstack/__init__.py +0 -0
  2. simstack/_version.py +24 -0
  3. simstack/core/__init__.py +0 -0
  4. simstack/core/artifacts.py +406 -0
  5. simstack/core/asnyc_helper.py +14 -0
  6. simstack/core/config_file.py +60 -0
  7. simstack/core/context.py +283 -0
  8. simstack/core/definitions.py +22 -0
  9. simstack/core/engine.py +115 -0
  10. simstack/core/find_simstack_modules.py +41 -0
  11. simstack/core/hash.py +179 -0
  12. simstack/core/node.py +1137 -0
  13. simstack/core/node_claim.py +31 -0
  14. simstack/core/node_runner.py +362 -0
  15. simstack/core/process_results.py +102 -0
  16. simstack/core/recompute_artifacts.py +117 -0
  17. simstack/core/reset_db.py +12 -0
  18. simstack/core/resource_assignment.py +222 -0
  19. simstack/core/resources.py +90 -0
  20. simstack/core/route_table.py +24 -0
  21. simstack/core/run_docker.py +109 -0
  22. simstack/core/run_node.py +71 -0
  23. simstack/core/runner.py +169 -0
  24. simstack/core/services/__init__.py +11 -0
  25. simstack/core/services/base_service.py +191 -0
  26. simstack/core/services/file_transfer_service.py +117 -0
  27. simstack/core/services/git_restart_service.py +12 -0
  28. simstack/core/services/git_uv_update_service.py +105 -0
  29. simstack/core/services/node_execution_service.py +182 -0
  30. simstack/core/services/resource_branch_monitor_service.py +71 -0
  31. simstack/core/services/runner_cleanup_service.py +149 -0
  32. simstack/core/services/runner_manager.py +163 -0
  33. simstack/core/services/runner_status_service.py +10 -0
  34. simstack/core/services/slurm_status_service.py +67 -0
  35. simstack/core/services/stop_check_service.py +27 -0
  36. simstack/core/services/timeout_restart_service.py +26 -0
  37. simstack/core/simstack_result.py +39 -0
  38. simstack/core/submit_node.py +217 -0
  39. simstack/core/task_id.py +22 -0
  40. simstack/methods/__init__.py +0 -0
  41. simstack/methods/archive_files.py +292 -0
  42. simstack/methods/fire_and_forget_runner.py +113 -0
  43. simstack/methods/generate_test.py +125 -0
  44. simstack/methods/get_file.py +26 -0
  45. simstack/methods/mass_runner.py +154 -0
  46. simstack/methods/multiple_executor.py +307 -0
  47. simstack/methods/switch_git.py +16 -0
  48. simstack/methods/upload_helpers.py +81 -0
  49. simstack/models/__init__.py +74 -0
  50. simstack/models/array_list.py +21 -0
  51. simstack/models/array_storage.py +83 -0
  52. simstack/models/artifact_models.py +127 -0
  53. simstack/models/base_lists.py +51 -0
  54. simstack/models/base_types.py +209 -0
  55. simstack/models/charts_artifact.py +690 -0
  56. simstack/models/dataset.py +451 -0
  57. simstack/models/dataset_metadata.py +390 -0
  58. simstack/models/datasettuple.py +587 -0
  59. simstack/models/file_instance.py +150 -0
  60. simstack/models/file_list.py +45 -0
  61. simstack/models/files.py +424 -0
  62. simstack/models/fire_and_forget_result.py +19 -0
  63. simstack/models/images2d.py +113 -0
  64. simstack/models/log_entry_model.py +37 -0
  65. simstack/models/models.py +52 -0
  66. simstack/models/named_data_reference.py +36 -0
  67. simstack/models/node_registry.py +96 -0
  68. simstack/models/pandas_model.py +239 -0
  69. simstack/models/parameters.py +402 -0
  70. simstack/models/pickle_models.py +300 -0
  71. simstack/models/project.py +13 -0
  72. simstack/models/resource_assignment.py +170 -0
  73. simstack/models/resource_definition.py +125 -0
  74. simstack/models/runner_model.py +39 -0
  75. simstack/models/simple_table.py +79 -0
  76. simstack/models/simstack_model.py +97 -0
  77. simstack/models/slurm_info.py +17 -0
  78. simstack/models/table_artifact.py +87 -0
  79. simstack/models/tag.py +12 -0
  80. simstack/models/unused_test_file.py +5 -0
  81. simstack/simstack_marker.com +1 -0
  82. simstack/static/docs/.buildinfo +4 -0
  83. simstack/static/docs/_images/GUI Concept.webp +0 -0
  84. simstack/static/docs/_modules/index.html +434 -0
  85. simstack/static/docs/_modules/simstack/core/artifacts.html +743 -0
  86. simstack/static/docs/_modules/simstack/core/asnyc_helper.html +330 -0
  87. simstack/static/docs/_modules/simstack/core/config_file.html +379 -0
  88. simstack/static/docs/_modules/simstack/core/definitions.html +341 -0
  89. simstack/static/docs/_modules/simstack/core/engine.html +437 -0
  90. simstack/static/docs/_modules/simstack/core/find_simstack_modules.html +360 -0
  91. simstack/static/docs/_modules/simstack/core/hash.html +528 -0
  92. simstack/static/docs/_modules/simstack/core/node.html +1392 -0
  93. simstack/static/docs/_modules/simstack/core/node_claim.html +347 -0
  94. simstack/static/docs/_modules/simstack/core/node_runner.html +711 -0
  95. simstack/static/docs/_modules/simstack/core/process_results.html +418 -0
  96. simstack/static/docs/_modules/simstack/core/recompute_artifacts.html +433 -0
  97. simstack/static/docs/_modules/simstack/core/reset_db.html +328 -0
  98. simstack/static/docs/_modules/simstack/core/resource_assignment.html +553 -0
  99. simstack/static/docs/_modules/simstack/core/resources.html +424 -0
  100. simstack/static/docs/_modules/simstack/core/route_table.html +349 -0
  101. simstack/static/docs/_modules/simstack/core/run_docker.html +425 -0
  102. simstack/static/docs/_modules/simstack/core/run_node.html +390 -0
  103. simstack/static/docs/_modules/simstack/core/runner.html +491 -0
  104. simstack/static/docs/_modules/simstack/core/services/base_service.html +522 -0
  105. simstack/static/docs/_modules/simstack/core/services/file_transfer_service.html +436 -0
  106. simstack/static/docs/_modules/simstack/core/services/git_restart_service.html +331 -0
  107. simstack/static/docs/_modules/simstack/core/services/git_uv_update_service.html +427 -0
  108. simstack/static/docs/_modules/simstack/core/services/node_execution_service.html +507 -0
  109. simstack/static/docs/_modules/simstack/core/services/resource_branch_monitor_service.html +390 -0
  110. simstack/static/docs/_modules/simstack/core/services/runner_cleanup_service.html +468 -0
  111. simstack/static/docs/_modules/simstack/core/services/runner_manager.html +485 -0
  112. simstack/static/docs/_modules/simstack/core/services/runner_status_service.html +329 -0
  113. simstack/static/docs/_modules/simstack/core/services/slurm_status_service.html +386 -0
  114. simstack/static/docs/_modules/simstack/core/services/stop_check_service.html +346 -0
  115. simstack/static/docs/_modules/simstack/core/services/timeout_restart_service.html +345 -0
  116. simstack/static/docs/_modules/simstack/core/simstack_result.html +355 -0
  117. simstack/static/docs/_modules/simstack/core/submit_node.html +536 -0
  118. simstack/static/docs/_modules/simstack/core/task_id.html +344 -0
  119. simstack/static/docs/_modules/simstack/methods/archive_files.html +598 -0
  120. simstack/static/docs/_modules/simstack/methods/fire_and_forget_runner.html +432 -0
  121. simstack/static/docs/_modules/simstack/methods/generate_test.html +443 -0
  122. simstack/static/docs/_modules/simstack/methods/get_file.html +342 -0
  123. simstack/static/docs/_modules/simstack/methods/mass_runner.html +476 -0
  124. simstack/static/docs/_modules/simstack/methods/multiple_executor.html +674 -0
  125. simstack/static/docs/_modules/simstack/methods/switch_git.html +332 -0
  126. simstack/static/docs/_modules/simstack/methods/upload_helpers.html +403 -0
  127. simstack/static/docs/_modules/simstack/models/array_list.html +337 -0
  128. simstack/static/docs/_modules/simstack/models/array_storage.html +417 -0
  129. simstack/static/docs/_modules/simstack/models/base_lists.html +374 -0
  130. simstack/static/docs/_modules/simstack/models/base_types.html +588 -0
  131. simstack/static/docs/_modules/simstack/models/charts_artifact.html +1035 -0
  132. simstack/static/docs/_modules/simstack/models/dataset.html +866 -0
  133. simstack/static/docs/_modules/simstack/models/dataset_metadata.html +711 -0
  134. simstack/static/docs/_modules/simstack/models/datasettuple.html +1005 -0
  135. simstack/static/docs/_modules/simstack/models/file_instance.html +472 -0
  136. simstack/static/docs/_modules/simstack/models/file_list.html +367 -0
  137. simstack/static/docs/_modules/simstack/models/files.html +773 -0
  138. simstack/static/docs/_modules/simstack/models/fire_and_forget_result.html +335 -0
  139. simstack/static/docs/_modules/simstack/models/images2d.html +447 -0
  140. simstack/static/docs/_modules/simstack/models/log_entry_model.html +356 -0
  141. simstack/static/docs/_modules/simstack/models/models.html +374 -0
  142. simstack/static/docs/_modules/simstack/models/named_data_reference.html +355 -0
  143. simstack/static/docs/_modules/simstack/models/node_registry.html +415 -0
  144. simstack/static/docs/_modules/simstack/models/pandas_model.html +570 -0
  145. simstack/static/docs/_modules/simstack/models/parameters.html +739 -0
  146. simstack/static/docs/_modules/simstack/models/pickle_models.html +631 -0
  147. simstack/static/docs/_modules/simstack/models/project.html +329 -0
  148. simstack/static/docs/_modules/simstack/models/resource_assignment.html +501 -0
  149. simstack/static/docs/_modules/simstack/models/resource_definition.html +477 -0
  150. simstack/static/docs/_modules/simstack/models/runner_model.html +361 -0
  151. simstack/static/docs/_modules/simstack/models/simple_table.html +413 -0
  152. simstack/static/docs/_modules/simstack/models/simstack_model.html +416 -0
  153. simstack/static/docs/_modules/simstack/models/slurm_info.html +333 -0
  154. simstack/static/docs/_modules/simstack/models/table_artifact.html +406 -0
  155. simstack/static/docs/_modules/simstack/models/tag.html +328 -0
  156. simstack/static/docs/_modules/simstack/tables/model_table.html +519 -0
  157. simstack/static/docs/_modules/simstack/tables/node_children.html +538 -0
  158. simstack/static/docs/_modules/simstack/tables/node_table.html +854 -0
  159. simstack/static/docs/_modules/simstack/tables/table_builder.html +642 -0
  160. simstack/static/docs/_modules/simstack/util/async_zip_utils.html +337 -0
  161. simstack/static/docs/_modules/simstack/util/b64mixin.html +339 -0
  162. simstack/static/docs/_modules/simstack/util/cleaned_json_schema.html +336 -0
  163. simstack/static/docs/_modules/simstack/util/config_reader.html +498 -0
  164. simstack/static/docs/_modules/simstack/util/custom_model_dump.html +391 -0
  165. simstack/static/docs/_modules/simstack/util/database_information.html +488 -0
  166. simstack/static/docs/_modules/simstack/util/db.html +1392 -0
  167. simstack/static/docs/_modules/simstack/util/db_logger.html +448 -0
  168. simstack/static/docs/_modules/simstack/util/default_from_dict.html +376 -0
  169. simstack/static/docs/_modules/simstack/util/directory_iterator.html +553 -0
  170. simstack/static/docs/_modules/simstack/util/docstring_parser.html +526 -0
  171. simstack/static/docs/_modules/simstack/util/file_hashing.html +394 -0
  172. simstack/static/docs/_modules/simstack/util/file_transfer_client.html +791 -0
  173. simstack/static/docs/_modules/simstack/util/files.html +733 -0
  174. simstack/static/docs/_modules/simstack/util/find_python_files.html +349 -0
  175. simstack/static/docs/_modules/simstack/util/generate_ui_schema.html +396 -0
  176. simstack/static/docs/_modules/simstack/util/generic_list_mixin.html +487 -0
  177. simstack/static/docs/_modules/simstack/util/get_module_path.html +336 -0
  178. simstack/static/docs/_modules/simstack/util/git_pull.html +381 -0
  179. simstack/static/docs/_modules/simstack/util/git_repository_status.html +411 -0
  180. simstack/static/docs/_modules/simstack/util/import_module.html +377 -0
  181. simstack/static/docs/_modules/simstack/util/importer.html +637 -0
  182. simstack/static/docs/_modules/simstack/util/init_data_source.html +357 -0
  183. simstack/static/docs/_modules/simstack/util/make_graph.html +423 -0
  184. simstack/static/docs/_modules/simstack/util/make_table.html +1171 -0
  185. simstack/static/docs/_modules/simstack/util/mappings.html +378 -0
  186. simstack/static/docs/_modules/simstack/util/minimal_route_finder.html +388 -0
  187. simstack/static/docs/_modules/simstack/util/mock_db.html +472 -0
  188. simstack/static/docs/_modules/simstack/util/module_path_checker.html +354 -0
  189. simstack/static/docs/_modules/simstack/util/object_list_mixin.html +649 -0
  190. simstack/static/docs/_modules/simstack/util/path_manager.html +566 -0
  191. simstack/static/docs/_modules/simstack/util/project_root_finder.html +357 -0
  192. simstack/static/docs/_modules/simstack/util/queue_watcher.html +593 -0
  193. simstack/static/docs/_modules/simstack/util/resource_config.html +532 -0
  194. simstack/static/docs/_modules/simstack/util/route_finder.html +390 -0
  195. simstack/static/docs/_modules/simstack/util/runner_util.html +487 -0
  196. simstack/static/docs/_modules/simstack/util/runner_utils.html +468 -0
  197. simstack/static/docs/_modules/simstack/util/safe_code_executor.html +510 -0
  198. simstack/static/docs/_modules/simstack/util/sample_config.html +414 -0
  199. simstack/static/docs/_modules/simstack/util/sensitive_content_filter.html +364 -0
  200. simstack/static/docs/_modules/simstack/util/setup_logging.html +365 -0
  201. simstack/static/docs/_modules/simstack/util/submit_to_watchdog.html +425 -0
  202. simstack/static/docs/_modules/simstack/util/toml_reader.html +482 -0
  203. simstack/static/docs/_modules/simstack/util/transform_file_name.html +357 -0
  204. simstack/static/docs/_modules/simstack/util/ui_tools.html +541 -0
  205. simstack/static/docs/_sources/architecture.rst.txt +25 -0
  206. simstack/static/docs/_sources/array_lists.rst.txt +60 -0
  207. simstack/static/docs/_sources/ci_pipeline.rst.txt +34 -0
  208. simstack/static/docs/_sources/data.rst.txt +175 -0
  209. simstack/static/docs/_sources/database_migrations.rst.txt +310 -0
  210. simstack/static/docs/_sources/datasets.rst.txt +93 -0
  211. simstack/static/docs/_sources/developer_guide.rst.txt +25 -0
  212. simstack/static/docs/_sources/docker_nodes.rst.txt +149 -0
  213. simstack/static/docs/_sources/files.rst.txt +159 -0
  214. simstack/static/docs/_sources/fire_and_forget_runner.rst.txt +67 -0
  215. simstack/static/docs/_sources/index.rst.txt +25 -0
  216. simstack/static/docs/_sources/installation.rst.txt +213 -0
  217. simstack/static/docs/_sources/lists.rst.txt +111 -0
  218. simstack/static/docs/_sources/mass_runner.rst.txt +91 -0
  219. simstack/static/docs/_sources/misc.rst.txt +106 -0
  220. simstack/static/docs/_sources/overview.rst.txt +91 -0
  221. simstack/static/docs/_sources/recovery.rst.txt +38 -0
  222. simstack/static/docs/_sources/resource_config.rst.txt +141 -0
  223. simstack/static/docs/_sources/simstack/modules/modules.rst.txt +7 -0
  224. simstack/static/docs/_sources/simstack/modules/simstack.core.rst.txt +205 -0
  225. simstack/static/docs/_sources/simstack/modules/simstack.core.services.rst.txt +109 -0
  226. simstack/static/docs/_sources/simstack/modules/simstack.methods.rst.txt +77 -0
  227. simstack/static/docs/_sources/simstack/modules/simstack.models.rst.txt +261 -0
  228. simstack/static/docs/_sources/simstack/modules/simstack.rst.txt +22 -0
  229. simstack/static/docs/_sources/simstack/modules/simstack.tables.rst.txt +45 -0
  230. simstack/static/docs/_sources/simstack/modules/simstack.util.rst.txt +381 -0
  231. simstack/static/docs/_sources/submit.rst.txt +141 -0
  232. simstack/static/docs/_sources/tables.rst.txt +50 -0
  233. simstack/static/docs/_sources/testing_complex_nodes.rst.txt +92 -0
  234. simstack/static/docs/_sources/ui_documentation.rst.txt +272 -0
  235. simstack/static/docs/_sources/workflows.rst.txt +446 -0
  236. simstack/static/docs/_static/base-stemmer.js +476 -0
  237. simstack/static/docs/_static/basic.css +906 -0
  238. simstack/static/docs/_static/cobalt2-theme.css +1 -0
  239. simstack/static/docs/_static/debug.css +69 -0
  240. simstack/static/docs/_static/docs-build-badge.css +51 -0
  241. simstack/static/docs/_static/docs-build-badge.js +51 -0
  242. simstack/static/docs/_static/doctools.js +150 -0
  243. simstack/static/docs/_static/documentation_options.js +13 -0
  244. simstack/static/docs/_static/english-stemmer.js +1066 -0
  245. simstack/static/docs/_static/file.png +0 -0
  246. simstack/static/docs/_static/language_data.js +13 -0
  247. simstack/static/docs/_static/minus.png +0 -0
  248. simstack/static/docs/_static/plus.png +0 -0
  249. simstack/static/docs/_static/pygments.css +250 -0
  250. simstack/static/docs/_static/scripts/furo-extensions.js +0 -0
  251. simstack/static/docs/_static/scripts/furo.js +3 -0
  252. simstack/static/docs/_static/scripts/furo.js.LICENSE.txt +7 -0
  253. simstack/static/docs/_static/scripts/furo.js.map +1 -0
  254. simstack/static/docs/_static/searchtools.js +693 -0
  255. simstack/static/docs/_static/skeleton.css +296 -0
  256. simstack/static/docs/_static/sphinx_highlight.js +159 -0
  257. simstack/static/docs/_static/styles/furo-extensions.css +2 -0
  258. simstack/static/docs/_static/styles/furo-extensions.css.map +1 -0
  259. simstack/static/docs/_static/styles/furo.css +2 -0
  260. simstack/static/docs/_static/styles/furo.css.map +1 -0
  261. simstack/static/docs/architecture.html +340 -0
  262. simstack/static/docs/array_lists.html +678 -0
  263. simstack/static/docs/ci_pipeline.html +385 -0
  264. simstack/static/docs/data.html +545 -0
  265. simstack/static/docs/database_migrations.html +652 -0
  266. simstack/static/docs/datasets.html +1268 -0
  267. simstack/static/docs/developer_guide.html +359 -0
  268. simstack/static/docs/docker_nodes.html +503 -0
  269. simstack/static/docs/files.html +718 -0
  270. simstack/static/docs/fire_and_forget_runner.html +641 -0
  271. simstack/static/docs/genindex.html +6390 -0
  272. simstack/static/docs/index.html +516 -0
  273. simstack/static/docs/installation.html +553 -0
  274. simstack/static/docs/lists.html +478 -0
  275. simstack/static/docs/mass_runner.html +481 -0
  276. simstack/static/docs/misc.html +454 -0
  277. simstack/static/docs/objects.inv +0 -0
  278. simstack/static/docs/overview.html +469 -0
  279. simstack/static/docs/py-modindex.html +1111 -0
  280. simstack/static/docs/recovery.html +371 -0
  281. simstack/static/docs/resource_config.html +590 -0
  282. simstack/static/docs/search.html +327 -0
  283. simstack/static/docs/searchindex.js +1 -0
  284. simstack/static/docs/simstack/modules/modules.html +489 -0
  285. simstack/static/docs/simstack/modules/simstack.core.html +1883 -0
  286. simstack/static/docs/simstack/modules/simstack.core.services.html +698 -0
  287. simstack/static/docs/simstack/modules/simstack.html +3060 -0
  288. simstack/static/docs/simstack/modules/simstack.methods.html +1150 -0
  289. simstack/static/docs/simstack/modules/simstack.models.html +13113 -0
  290. simstack/static/docs/simstack/modules/simstack.tables.html +635 -0
  291. simstack/static/docs/simstack/modules/simstack.util.html +3320 -0
  292. simstack/static/docs/submit.html +487 -0
  293. simstack/static/docs/tables.html +539 -0
  294. simstack/static/docs/testing_complex_nodes.html +458 -0
  295. simstack/static/docs/ui_documentation.html +788 -0
  296. simstack/static/docs/workflows.html +766 -0
  297. simstack/tables/__init__.py +0 -0
  298. simstack/tables/model_table.py +191 -0
  299. simstack/tables/node_children.py +219 -0
  300. simstack/tables/node_table.py +517 -0
  301. simstack/tables/table_builder.py +314 -0
  302. simstack/util/__init__.py +0 -0
  303. simstack/util/async_zip_utils.py +18 -0
  304. simstack/util/b64mixin.py +23 -0
  305. simstack/util/cleaned_json_schema.py +20 -0
  306. simstack/util/config_reader.py +179 -0
  307. simstack/util/custom_model_dump.py +75 -0
  308. simstack/util/database_information.py +160 -0
  309. simstack/util/db.py +1013 -0
  310. simstack/util/db_logger.py +117 -0
  311. simstack/util/default_from_dict.py +57 -0
  312. simstack/util/directory_iterator.py +216 -0
  313. simstack/util/docstring_parser.py +186 -0
  314. simstack/util/file_hashing.py +72 -0
  315. simstack/util/file_transfer_client.py +427 -0
  316. simstack/util/files.py +396 -0
  317. simstack/util/find_python_files.py +33 -0
  318. simstack/util/generate_ui_schema.py +80 -0
  319. simstack/util/generic_list_mixin.py +120 -0
  320. simstack/util/geometry_utils.py +118 -0
  321. simstack/util/get_module_path.py +20 -0
  322. simstack/util/git_pull.py +65 -0
  323. simstack/util/git_repository_status.py +95 -0
  324. simstack/util/import_module.py +63 -0
  325. simstack/util/importer.py +312 -0
  326. simstack/util/init_data_source.py +41 -0
  327. simstack/util/make_graph.py +107 -0
  328. simstack/util/make_table.py +837 -0
  329. simstack/util/mappings.py +41 -0
  330. simstack/util/minimal_route_finder.py +72 -0
  331. simstack/util/mock_db.py +144 -0
  332. simstack/util/module_path_checker.py +38 -0
  333. simstack/util/object_list_mixin.py +282 -0
  334. simstack/util/path_manager.py +223 -0
  335. simstack/util/project_root_finder.py +41 -0
  336. simstack/util/queue_watcher.py +262 -0
  337. simstack/util/resource_config.py +198 -0
  338. simstack/util/route_finder.py +74 -0
  339. simstack/util/runner_util.py +166 -0
  340. simstack/util/runner_utils.py +143 -0
  341. simstack/util/safe_code_executor.py +194 -0
  342. simstack/util/sample_config.py +98 -0
  343. simstack/util/sensitive_content_filter.py +48 -0
  344. simstack/util/setup_logging.py +49 -0
  345. simstack/util/submit_to_watchdog.py +106 -0
  346. simstack/util/toml_reader.py +142 -0
  347. simstack/util/transform_file_name.py +41 -0
  348. simstack/util/ui_tools.py +213 -0
  349. simstack-2.0.1.dist-info/METADATA +454 -0
  350. simstack-2.0.1.dist-info/RECORD +354 -0
  351. simstack-2.0.1.dist-info/WHEEL +5 -0
  352. simstack-2.0.1.dist-info/entry_points.txt +12 -0
  353. simstack-2.0.1.dist-info/licenses/LICENSE +21 -0
  354. simstack-2.0.1.dist-info/top_level.txt +1 -0
simstack/static/docs/_modules/simstack/methods/archive_files.html ADDED
@@ -0,0 +1,598 @@
1
+ <!doctype html>
2
+ <html class="no-js" lang="en" data-content_root="../../../">
3
+ <head><meta charset="utf-8">
4
+ <meta name="viewport" content="width=device-width,initial-scale=1">
5
+ <meta name="color-scheme" content="light dark"><link rel="index" title="Index" href="../../../genindex.html"><link rel="search" title="Search" href="../../../search.html">
6
+
7
+ <!-- Generated with Sphinx 9.1.0 and Furo 2025.12.19 -->
8
+ <title>simstack.methods.archive_files - SimStack II documentation</title>
9
+ <link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=d111a655" />
10
+ <link rel="stylesheet" type="text/css" href="../../../_static/styles/furo.css?v=7bdb33bb" />
11
+ <link rel="stylesheet" type="text/css" href="../../../_static/styles/furo-extensions.css?v=8dab3a3b" />
12
+ <link rel="stylesheet" type="text/css" href="../../../_static/cobalt2-theme.css?v=2d651046" />
13
+ <link rel="stylesheet" type="text/css" href="../../../_static/docs-build-badge.css?v=803faff8" />
14
+
15
+
16
+
17
+
18
+ <style>
19
+ body {
20
+ --color-code-background: #f2f2f2;
21
+ --color-code-foreground: #1e1e1e;
22
+
23
+ }
24
+ @media not print {
25
+ body[data-theme="dark"] {
26
+ --color-code-background: #202020;
27
+ --color-code-foreground: #d0d0d0;
28
+ --color-background-primary: #132738;
29
+ --color-background-secondary: #1e3a52;
30
+ --color-foreground-primary: #ffffff;
31
+ --color-foreground-secondary: #bac8d3;
32
+ --color-brand-primary: #22d5ff;
33
+ --color-brand-content: #0088ff;
34
+
35
+ }
36
+ @media (prefers-color-scheme: dark) {
37
+ body:not([data-theme="light"]) {
38
+ --color-code-background: #202020;
39
+ --color-code-foreground: #d0d0d0;
40
+ --color-background-primary: #132738;
41
+ --color-background-secondary: #1e3a52;
42
+ --color-foreground-primary: #ffffff;
43
+ --color-foreground-secondary: #bac8d3;
44
+ --color-brand-primary: #22d5ff;
45
+ --color-brand-content: #0088ff;
46
+
47
+ }
48
+ }
49
+ }
50
+ </style></head>
51
+ <body>
52
+
53
+ <script>
54
+ document.body.dataset.theme = localStorage.getItem("theme") || "auto";
55
+ </script>
56
+
57
+
58
+ <svg xmlns="http://www.w3.org/2000/svg" style="display: none;">
59
+ <symbol id="svg-toc" viewBox="0 0 24 24">
60
+ <title>Contents</title>
61
+ <svg stroke="currentColor" fill="currentColor" stroke-width="0" viewBox="0 0 1024 1024">
62
+ <path d="M408 442h480c4.4 0 8-3.6 8-8v-56c0-4.4-3.6-8-8-8H408c-4.4 0-8 3.6-8 8v56c0 4.4 3.6 8 8 8zm-8 204c0 4.4 3.6 8 8 8h480c4.4 0 8-3.6 8-8v-56c0-4.4-3.6-8-8-8H408c-4.4 0-8 3.6-8 8v56zm504-486H120c-4.4 0-8 3.6-8 8v56c0 4.4 3.6 8 8 8h784c4.4 0 8-3.6 8-8v-56c0-4.4-3.6-8-8-8zm0 632H120c-4.4 0-8 3.6-8 8v56c0 4.4 3.6 8 8 8h784c4.4 0 8-3.6 8-8v-56c0-4.4-3.6-8-8-8zM115.4 518.9L271.7 642c5.8 4.6 14.4.5 14.4-6.9V388.9c0-7.4-8.5-11.5-14.4-6.9L115.4 505.1a8.74 8.74 0 0 0 0 13.8z"/>
63
+ </svg>
64
+ </symbol>
65
+ <symbol id="svg-menu" viewBox="0 0 24 24">
66
+ <title>Menu</title>
67
+ <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
68
+ stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather-menu">
69
+ <line x1="3" y1="12" x2="21" y2="12"></line>
70
+ <line x1="3" y1="6" x2="21" y2="6"></line>
71
+ <line x1="3" y1="18" x2="21" y2="18"></line>
72
+ </svg>
73
+ </symbol>
74
+ <symbol id="svg-arrow-right" viewBox="0 0 24 24">
75
+ <title>Expand</title>
76
+ <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
77
+ stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather-chevron-right">
78
+ <polyline points="9 18 15 12 9 6"></polyline>
79
+ </svg>
80
+ </symbol>
81
+ <symbol id="svg-sun" viewBox="0 0 24 24">
82
+ <title>Light mode</title>
83
+ <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
84
+ stroke-width="1" stroke-linecap="round" stroke-linejoin="round" class="feather-sun">
85
+ <circle cx="12" cy="12" r="5"></circle>
86
+ <line x1="12" y1="1" x2="12" y2="3"></line>
87
+ <line x1="12" y1="21" x2="12" y2="23"></line>
88
+ <line x1="4.22" y1="4.22" x2="5.64" y2="5.64"></line>
89
+ <line x1="18.36" y1="18.36" x2="19.78" y2="19.78"></line>
90
+ <line x1="1" y1="12" x2="3" y2="12"></line>
91
+ <line x1="21" y1="12" x2="23" y2="12"></line>
92
+ <line x1="4.22" y1="19.78" x2="5.64" y2="18.36"></line>
93
+ <line x1="18.36" y1="5.64" x2="19.78" y2="4.22"></line>
94
+ </svg>
95
+ </symbol>
96
+ <symbol id="svg-moon" viewBox="0 0 24 24">
97
+ <title>Dark mode</title>
98
+ <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
99
+ stroke-width="1" stroke-linecap="round" stroke-linejoin="round" class="icon-tabler-moon">
100
+ <path stroke="none" d="M0 0h24v24H0z" fill="none" />
101
+ <path d="M12 3c.132 0 .263 0 .393 0a7.5 7.5 0 0 0 7.92 12.446a9 9 0 1 1 -8.313 -12.454z" />
102
+ </svg>
103
+ </symbol>
104
+ <symbol id="svg-sun-with-moon" viewBox="0 0 24 24">
105
+ <title>Auto light/dark, in light mode</title>
106
+ <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
107
+ stroke-width="1" stroke-linecap="round" stroke-linejoin="round"
108
+ class="icon-custom-derived-from-feather-sun-and-tabler-moon">
109
+ <path style="opacity: 50%" d="M 5.411 14.504 C 5.471 14.504 5.532 14.504 5.591 14.504 C 3.639 16.319 4.383 19.569 6.931 20.352 C 7.693 20.586 8.512 20.551 9.25 20.252 C 8.023 23.207 4.056 23.725 2.11 21.184 C 0.166 18.642 1.702 14.949 4.874 14.536 C 5.051 14.512 5.231 14.5 5.411 14.5 L 5.411 14.504 Z"/>
110
+ <line x1="14.5" y1="3.25" x2="14.5" y2="1.25"/>
111
+ <line x1="14.5" y1="15.85" x2="14.5" y2="17.85"/>
112
+ <line x1="10.044" y1="5.094" x2="8.63" y2="3.68"/>
113
+ <line x1="19" y1="14.05" x2="20.414" y2="15.464"/>
114
+ <line x1="8.2" y1="9.55" x2="6.2" y2="9.55"/>
115
+ <line x1="20.8" y1="9.55" x2="22.8" y2="9.55"/>
116
+ <line x1="10.044" y1="14.006" x2="8.63" y2="15.42"/>
117
+ <line x1="19" y1="5.05" x2="20.414" y2="3.636"/>
118
+ <circle cx="14.5" cy="9.55" r="3.6"/>
119
+ </svg>
120
+ </symbol>
121
+ <symbol id="svg-moon-with-sun" viewBox="0 0 24 24">
122
+ <title>Auto light/dark, in dark mode</title>
123
+ <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
124
+ stroke-width="1" stroke-linecap="round" stroke-linejoin="round"
125
+ class="icon-custom-derived-from-feather-sun-and-tabler-moon">
126
+ <path d="M 8.282 7.007 C 8.385 7.007 8.494 7.007 8.595 7.007 C 5.18 10.184 6.481 15.869 10.942 17.24 C 12.275 17.648 13.706 17.589 15 17.066 C 12.851 22.236 5.91 23.143 2.505 18.696 C -0.897 14.249 1.791 7.786 7.342 7.063 C 7.652 7.021 7.965 7 8.282 7 L 8.282 7.007 Z"/>
127
+ <line style="opacity: 50%" x1="18" y1="3.705" x2="18" y2="2.5"/>
128
+ <line style="opacity: 50%" x1="18" y1="11.295" x2="18" y2="12.5"/>
129
+ <line style="opacity: 50%" x1="15.316" y1="4.816" x2="14.464" y2="3.964"/>
130
+ <line style="opacity: 50%" x1="20.711" y1="10.212" x2="21.563" y2="11.063"/>
131
+ <line style="opacity: 50%" x1="14.205" y1="7.5" x2="13.001" y2="7.5"/>
132
+ <line style="opacity: 50%" x1="21.795" y1="7.5" x2="23" y2="7.5"/>
133
+ <line style="opacity: 50%" x1="15.316" y1="10.184" x2="14.464" y2="11.036"/>
134
+ <line style="opacity: 50%" x1="20.711" y1="4.789" x2="21.563" y2="3.937"/>
135
+ <circle style="opacity: 50%" cx="18" cy="7.5" r="2.169"/>
136
+ </svg>
137
+ </symbol>
138
+ <symbol id="svg-pencil" viewBox="0 0 24 24">
139
+ <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
140
+ stroke-width="1" stroke-linecap="round" stroke-linejoin="round" class="icon-tabler-pencil-code">
141
+ <path d="M4 20h4l10.5 -10.5a2.828 2.828 0 1 0 -4 -4l-10.5 10.5v4" />
142
+ <path d="M13.5 6.5l4 4" />
143
+ <path d="M20 21l2 -2l-2 -2" />
144
+ <path d="M17 17l-2 2l2 2" />
145
+ </svg>
146
+ </symbol>
147
+ <symbol id="svg-eye" viewBox="0 0 24 24">
148
+ <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
149
+ stroke-width="1" stroke-linecap="round" stroke-linejoin="round" class="icon-tabler-eye-code">
150
+ <path stroke="none" d="M0 0h24v24H0z" fill="none" />
151
+ <path d="M10 12a2 2 0 1 0 4 0a2 2 0 0 0 -4 0" />
152
+ <path
153
+ d="M11.11 17.958c-3.209 -.307 -5.91 -2.293 -8.11 -5.958c2.4 -4 5.4 -6 9 -6c3.6 0 6.6 2 9 6c-.21 .352 -.427 .688 -.647 1.008" />
154
+ <path d="M20 21l2 -2l-2 -2" />
155
+ <path d="M17 17l-2 2l2 2" />
156
+ </svg>
157
+ </symbol>
158
+ </svg>
159
+
160
+ <input type="checkbox" class="sidebar-toggle" name="__navigation" id="__navigation" aria-label="Toggle site navigation sidebar">
161
+ <input type="checkbox" class="sidebar-toggle" name="__toc" id="__toc" aria-label="Toggle table of contents sidebar">
162
+ <label class="overlay sidebar-overlay" for="__navigation"></label>
163
+ <label class="overlay toc-overlay" for="__toc"></label>
164
+
165
+ <a class="skip-to-content muted-link" href="#furo-main-content">Skip to content</a>
166
+
167
+
168
+
169
+ <div class="page">
170
+ <header class="mobile-header">
171
+ <div class="header-left">
172
+ <label class="nav-overlay-icon" for="__navigation">
173
+ <span class="icon"><svg><use href="#svg-menu"></use></svg></span>
174
+ </label>
175
+ </div>
176
+ <div class="header-center">
177
+ <a href="../../../index.html"><div class="brand">SimStack II documentation</div></a>
178
+ </div>
179
+ <div class="header-right">
180
+ <div class="theme-toggle-container theme-toggle-header">
181
+ <button class="theme-toggle" aria-label="Toggle Light / Dark / Auto color theme">
182
+ <svg class="theme-icon-when-auto-light"><use href="#svg-sun-with-moon"></use></svg>
183
+ <svg class="theme-icon-when-auto-dark"><use href="#svg-moon-with-sun"></use></svg>
184
+ <svg class="theme-icon-when-dark"><use href="#svg-moon"></use></svg>
185
+ <svg class="theme-icon-when-light"><use href="#svg-sun"></use></svg>
186
+ </button>
187
+ </div>
188
+ <label class="toc-overlay-icon toc-header-icon no-toc" for="__toc">
189
+ <span class="icon"><svg><use href="#svg-toc"></use></svg></span>
190
+ </label>
191
+ </div>
192
+ </header>
193
+ <aside class="sidebar-drawer">
194
+ <div class="sidebar-container">
195
+
196
+ <div class="sidebar-sticky"><a class="sidebar-brand" href="../../../index.html">
197
+
198
+
199
+ </a><form class="sidebar-search-container" method="get" action="../../../search.html" role="search">
200
+ <input class="sidebar-search" placeholder="Search" name="q" aria-label="Search">
201
+ <input type="hidden" name="check_keywords" value="yes">
202
+ <input type="hidden" name="area" value="default">
203
+ </form>
204
+ <div id="searchbox"></div><div class="sidebar-scroll"><div class="sidebar-tree">
205
+ <p class="caption" role="heading"><span class="caption-text">Contents:</span></p>
206
+ <ul>
207
+ <li class="toctree-l1"><a class="reference internal" href="../../../overview.html">Overview</a></li>
208
+ <li class="toctree-l1"><a class="reference internal" href="../../../workflows.html">Workflows</a></li>
209
+ <li class="toctree-l1"><a class="reference internal" href="../../../ui_documentation.html">Graphical User Interface</a></li>
210
+ <li class="toctree-l1 has-children"><a class="reference internal" href="../../../developer_guide.html">Developer Guide</a><input aria-label="Toggle navigation of Developer Guide" class="toctree-checkbox" id="toctree-checkbox-1" name="toctree-checkbox-1" role="switch" type="checkbox"/><label for="toctree-checkbox-1"><span class="icon"><svg><use href="#svg-arrow-right"></use></svg></span></label><ul>
211
+ <li class="toctree-l2"><a class="reference internal" href="../../../data.html">Persisting and Visualizing Data</a></li>
212
+ <li class="toctree-l2"><a class="reference internal" href="../../../datasets.html">Datasets</a></li>
213
+ <li class="toctree-l2"><a class="reference internal" href="../../../lists.html">List Mixins</a></li>
214
+ <li class="toctree-l2"><a class="reference internal" href="../../../array_lists.html">Array Lists</a></li>
215
+ <li class="toctree-l2"><a class="reference internal" href="../../../tables.html">Tables</a></li>
216
+ <li class="toctree-l2"><a class="reference internal" href="../../../files.html">File Management</a></li>
217
+ <li class="toctree-l2"><a class="reference internal" href="../../../resource_config.html">Program Execution</a></li>
218
+ <li class="toctree-l2"><a class="reference internal" href="../../../mass_runner.html">MassRunner</a></li>
219
+ <li class="toctree-l2"><a class="reference internal" href="../../../fire_and_forget_runner.html">FireAndForgetRunner</a></li>
220
+ <li class="toctree-l2"><a class="reference internal" href="../../../docker_nodes.html">Creating Docker Containers for SimStack Nodes</a></li>
221
+ <li class="toctree-l2"><a class="reference internal" href="../../../testing_complex_nodes.html">Testing Complex Nodes</a></li>
222
+ <li class="toctree-l2"><a class="reference internal" href="../../../database_migrations.html">Database Migrations</a></li>
223
+ <li class="toctree-l2"><a class="reference internal" href="../../../ci_pipeline.html">CI Pipeline and Documentation Artifacts</a></li>
224
+ <li class="toctree-l2"><a class="reference internal" href="../../../misc.html">Miscellaneous</a></li>
225
+ </ul>
226
+ </li>
227
+ <li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Simstack II Setup</a></li>
228
+ <li class="toctree-l1 has-children"><a class="reference internal" href="../../../simstack/modules/modules.html">simstack</a><input aria-label="Toggle navigation of simstack" class="toctree-checkbox" id="toctree-checkbox-2" name="toctree-checkbox-2" role="switch" type="checkbox"/><label for="toctree-checkbox-2"><span class="icon"><svg><use href="#svg-arrow-right"></use></svg></span></label><ul>
229
+ <li class="toctree-l2 has-children"><a class="reference internal" href="../../../simstack/modules/simstack.html">simstack package</a><input aria-label="Toggle navigation of simstack package" class="toctree-checkbox" id="toctree-checkbox-3" name="toctree-checkbox-3" role="switch" type="checkbox"/><label for="toctree-checkbox-3"><span class="icon"><svg><use href="#svg-arrow-right"></use></svg></span></label><ul>
230
+ <li class="toctree-l3 has-children"><a class="reference internal" href="../../../simstack/modules/simstack.core.html">simstack.core package</a><input aria-label="Toggle navigation of simstack.core package" class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" role="switch" type="checkbox"/><label for="toctree-checkbox-4"><span class="icon"><svg><use href="#svg-arrow-right"></use></svg></span></label><ul>
231
+ <li class="toctree-l4"><a class="reference internal" href="../../../simstack/modules/simstack.core.services.html">simstack.core.services package</a></li>
232
+ </ul>
233
+ </li>
234
+ <li class="toctree-l3"><a class="reference internal" href="../../../simstack/modules/simstack.methods.html">simstack.methods package</a></li>
235
+ <li class="toctree-l3"><a class="reference internal" href="../../../simstack/modules/simstack.models.html">simstack.models package</a></li>
236
+ <li class="toctree-l3"><a class="reference internal" href="../../../simstack/modules/simstack.tables.html">simstack.tables package</a></li>
237
+ <li class="toctree-l3"><a class="reference internal" href="../../../simstack/modules/simstack.util.html">simstack.util package</a></li>
238
+ </ul>
239
+ </li>
240
+ </ul>
241
+ </li>
242
+ </ul>
243
+
244
+ </div>
245
+ </div>
246
+
247
+ </div>
248
+
249
+ </div>
250
+ </aside>
251
+ <div class="main">
252
+ <div class="content">
253
+ <div class="article-container">
254
+ <a href="#" class="back-to-top muted-link">
255
+ <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24">
256
+ <path d="M13 20h-2V8l-5.5 5.5-1.42-1.42L12 4.16l7.92 7.92-1.42 1.42L13 8v12z"></path>
257
+ </svg>
258
+ <span>Back to top</span>
259
+ </a>
260
+ <div class="content-icon-container">
261
+ <div class="theme-toggle-container theme-toggle-content">
262
+ <button class="theme-toggle" aria-label="Toggle Light / Dark / Auto color theme">
263
+ <svg class="theme-icon-when-auto-light"><use href="#svg-sun-with-moon"></use></svg>
264
+ <svg class="theme-icon-when-auto-dark"><use href="#svg-moon-with-sun"></use></svg>
265
+ <svg class="theme-icon-when-dark"><use href="#svg-moon"></use></svg>
266
+ <svg class="theme-icon-when-light"><use href="#svg-sun"></use></svg>
267
+ </button>
268
+ </div>
269
+ <label class="toc-overlay-icon toc-content-icon no-toc" for="__toc">
270
+ <span class="icon"><svg><use href="#svg-toc"></use></svg></span>
271
+ </label>
272
+ </div>
273
+ <article role="main" id="furo-main-content">
274
+ <h1>Source code for simstack.methods.archive_files</h1><div class="highlight"><pre>
275
+ <span></span><span class="kn">import</span><span class="w"> </span><span class="nn">asyncio</span>
276
+ <span class="kn">import</span><span class="w"> </span><span class="nn">inspect</span>
277
+ <span class="kn">import</span><span class="w"> </span><span class="nn">shutil</span>
278
+ <span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span>
279
+ <span class="kn">from</span><span class="w"> </span><span class="nn">pathlib</span><span class="w"> </span><span class="kn">import</span> <span class="n">Path</span>
280
+ <span class="kn">from</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="kn">import</span> <span class="n">List</span><span class="p">,</span> <span class="n">Callable</span><span class="p">,</span> <span class="n">Any</span><span class="p">,</span> <span class="n">Optional</span>
281
+
282
+ <span class="kn">from</span><span class="w"> </span><span class="nn">docutils.nodes</span><span class="w"> </span><span class="kn">import</span> <span class="n">field_name</span>
283
+ <span class="kn">from</span><span class="w"> </span><span class="nn">odmantic</span><span class="w"> </span><span class="kn">import</span> <span class="n">Model</span><span class="p">,</span> <span class="n">Field</span>
284
+
285
+ <span class="kn">from</span><span class="w"> </span><span class="nn">simstack.core.context</span><span class="w"> </span><span class="kn">import</span> <span class="n">context</span>
286
+ <span class="kn">from</span><span class="w"> </span><span class="nn">simstack.core.node</span><span class="w"> </span><span class="kn">import</span> <span class="n">node</span>
287
+ <span class="kn">from</span><span class="w"> </span><span class="nn">simstack.models</span><span class="w"> </span><span class="kn">import</span> <span class="n">FileStack</span><span class="p">,</span> <span class="n">FileList</span><span class="p">,</span> <span class="n">simstack_model</span>
288
+ <span class="kn">from</span><span class="w"> </span><span class="nn">simstack.models.base_lists</span><span class="w"> </span><span class="kn">import</span> <span class="n">BooleanDataList</span>
289
+ <span class="kn">from</span><span class="w"> </span><span class="nn">simstack.models.file_instance</span><span class="w"> </span><span class="kn">import</span> <span class="n">FileInstance</span>
290
+
291
+
292
+ <div class="viewcode-block" id="archive_one_file">
293
+ <a class="viewcode-back" href="../../../simstack/modules/simstack.methods.html#simstack.methods.archive_files.archive_one_file">[docs]</a>
294
+ <span class="k">def</span><span class="w"> </span><span class="nf">archive_one_file</span><span class="p">(</span><span class="n">file_stack</span><span class="p">:</span> <span class="n">FileStack</span><span class="p">,</span><span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
295
+ <span class="n">node_runner</span> <span class="o">=</span> <span class="n">kwargs</span><span class="p">[</span><span class="s2">&quot;node_runner&quot;</span><span class="p">]</span>
296
+ <span class="n">node_runner</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Archiving file: </span><span class="si">{</span><span class="n">file_stack</span><span class="o">.</span><span class="n">name</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
297
+
298
+ <span class="n">config</span> <span class="o">=</span> <span class="n">context</span><span class="o">.</span><span class="n">resource_config</span><span class="o">.</span><span class="n">get_program</span><span class="p">(</span><span class="s2">&quot;archive_one_file&quot;</span><span class="p">)</span>
299
+ <span class="k">if</span> <span class="n">config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;archive_path&quot;</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
300
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;archive_path not set in config&quot;</span><span class="p">)</span>
301
+ <span class="n">archive_path</span> <span class="o">=</span> <span class="n">Path</span><span class="p">(</span><span class="n">config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;archive_path&quot;</span><span class="p">))</span>
302
+ <span class="n">full_path</span> <span class="o">=</span> <span class="n">archive_path</span> <span class="o">/</span> <span class="nb">str</span><span class="p">(</span><span class="n">file_stack</span><span class="o">.</span><span class="n">id</span><span class="p">)</span>
303
+ <span class="n">full_path</span><span class="o">.</span><span class="n">mkdir</span><span class="p">(</span><span class="n">parents</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">exist_ok</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
304
+ <span class="k">if</span> <span class="n">full_path</span><span class="o">.</span><span class="n">exists</span><span class="p">()</span> <span class="ow">and</span> <span class="n">full_path</span><span class="o">.</span><span class="n">is_dir</span><span class="p">():</span>
305
+ <span class="k">try</span><span class="p">:</span>
306
+ <span class="n">archive_location</span> <span class="o">=</span> <span class="n">file_stack</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">local_dir</span><span class="o">=</span><span class="n">full_path</span><span class="p">)</span>
307
+ <span class="n">archive_file_instance</span> <span class="o">=</span> <span class="n">FileInstance</span><span class="o">.</span><span class="n">from_local_file</span><span class="p">(</span><span class="n">archive_location</span><span class="p">)</span>
308
+ <span class="n">node_runner</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Archived file to </span><span class="si">{</span><span class="n">archive_file_instance</span><span class="o">.</span><span class="n">path</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
309
+ <span class="n">file_stack</span><span class="o">.</span><span class="n">locations</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">archive_file_instance</span><span class="p">)</span>
310
+ <span class="k">return</span> <span class="kc">True</span>
311
+ <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
312
+ <span class="n">node_runner</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Error archiving file: </span><span class="si">{</span><span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">)</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
313
+ <span class="k">return</span> <span class="kc">False</span>
314
+ <span class="k">else</span><span class="p">:</span>
315
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Archive path </span><span class="si">{</span><span class="n">archive_path</span><span class="si">}</span><span class="s2"> does not exist or is not a directory&quot;</span><span class="p">)</span></div>
316
+
317
+
318
+ <div class="viewcode-block" id="archive_file">
319
+ <a class="viewcode-back" href="../../../simstack/modules/simstack.methods.html#simstack.methods.archive_files.archive_file">[docs]</a>
320
+ <span class="nd">@node</span>
321
+ <span class="k">async</span> <span class="k">def</span><span class="w"> </span><span class="nf">archive_file</span><span class="p">(</span><span class="n">file_stack</span><span class="p">:</span> <span class="n">FileStack</span><span class="p">,</span><span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
322
+ <span class="k">return</span> <span class="n">archive_one_file</span><span class="p">(</span><span class="n">file_stack</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span></div>
323
+
324
+
325
+ <div class="viewcode-block" id="archive_files">
326
+ <a class="viewcode-back" href="../../../simstack/modules/simstack.methods.html#simstack.methods.archive_files.archive_files">[docs]</a>
327
+ <span class="nd">@node</span>
328
+ <span class="k">def</span><span class="w"> </span><span class="nf">archive_files</span><span class="p">(</span><span class="n">file_list</span><span class="p">:</span> <span class="n">FileList</span><span class="p">,</span><span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
329
+ <span class="n">archive_results</span> <span class="o">=</span> <span class="n">BooleanDataList</span><span class="p">(</span><span class="n">field_name</span><span class="o">=</span><span class="s2">&quot;archive_result&quot;</span><span class="p">)</span>
330
+ <span class="k">for</span> <span class="n">file_stack</span> <span class="ow">in</span> <span class="n">file_list</span><span class="p">:</span>
331
+ <span class="n">archive_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">archive_one_file</span><span class="p">(</span><span class="n">file_stack</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">))</span>
332
+ <span class="k">return</span> <span class="n">archive_results</span></div>
333
+
334
+
335
+
336
+ <div class="viewcode-block" id="ArchiveConfig">
337
+ <a class="viewcode-back" href="../../../simstack/modules/simstack.methods.html#simstack.methods.archive_files.ArchiveConfig">[docs]</a>
338
+ <span class="nd">@simstack_model</span>
339
+ <span class="k">class</span><span class="w"> </span><span class="nc">ArchiveConfig</span><span class="p">(</span><span class="n">Model</span><span class="p">):</span>
340
+ <span class="n">call_paths</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span><span class="n">default_factory</span><span class="o">=</span><span class="nb">list</span><span class="p">)</span>
341
+ <span class="n">archive_resource</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span><span class="n">default</span><span class="o">=</span><span class="s2">&quot;archive_resource&quot;</span><span class="p">,</span> <span class="n">description</span><span class="o">=</span><span class="s2">&quot;Name of the resource where the files will be archived&quot;</span><span class="p">)</span>
342
+ <span class="n">in_memory</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span><span class="n">default</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">description</span><span class="o">=</span><span class="s2">&quot;Whether to archive files in memory&quot;</span><span class="p">)</span>
343
+ <span class="n">use_time_window</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span><span class="n">default</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">description</span><span class="o">=</span><span class="s2">&quot;Whether to archive files within a time window&quot;</span><span class="p">)</span>
344
+ <span class="n">start_date</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">datetime</span><span class="p">]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span><span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">description</span><span class="o">=</span><span class="s2">&quot;Start of the period to archive files for&quot;</span><span class="p">)</span>
345
+ <span class="n">end_date</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">datetime</span><span class="p">]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span><span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">description</span><span class="o">=</span><span class="s2">&quot;End end of the period to archive files for&quot;</span><span class="p">)</span>
346
+ <span class="n">include_patterns</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span><span class="n">default_factory</span><span class="o">=</span><span class="nb">list</span><span class="p">,</span> <span class="n">description</span><span class="o">=</span><span class="s2">&quot;List of file patterns to include&quot;</span><span class="p">)</span>
347
+ <span class="n">exclude_patterns</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span><span class="n">default_factory</span><span class="o">=</span><span class="nb">list</span><span class="p">,</span> <span class="n">description</span><span class="o">=</span><span class="s2">&quot;List of file patterns to exclude&quot;</span><span class="p">)</span>
348
+ <span class="n">min_size</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span><span class="n">default</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">description</span><span class="o">=</span><span class="s2">&quot;Minimum size of files to archive&quot;</span><span class="p">)</span>
349
+ <span class="n">filter_by_resource</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span><span class="n">default</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">description</span><span class="o">=</span><span class="s2">&quot;Whether to only archive files that have a location on the current resource&quot;</span><span class="p">)</span>
350
+
351
+ <span class="n">model_config</span> <span class="o">=</span> <span class="p">{</span><span class="s2">&quot;collection&quot;</span><span class="p">:</span> <span class="s2">&quot;archive_config&quot;</span><span class="p">}</span></div>
352
+
353
+
354
+
355
+ <div class="viewcode-block" id="archive_node">
356
+ <a class="viewcode-back" href="../../../simstack/modules/simstack.methods.html#simstack.methods.archive_files.archive_node">[docs]</a>
357
+ <span class="nd">@node</span>
358
+ <span class="k">async</span> <span class="k">def</span><span class="w"> </span><span class="nf">archive_node</span><span class="p">(</span><span class="n">archive_config</span><span class="p">:</span> <span class="n">ArchiveConfig</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
359
+ <span class="w"> </span><span class="sd">&quot;&quot;&quot;</span>
360
+ <span class="sd"> Archives FileStacks matching the criteria in archive_config and deletes local instances upon success.</span>
361
+
362
+ <span class="sd"> Args:</span>
363
+ <span class="sd"> archive_config (ArchiveConfig): Configuration for the archive process.</span>
364
+ <span class="sd"> **kwargs: Additional arguments.</span>
365
+ <span class="sd"> &quot;&quot;&quot;</span>
366
+ <span class="kn">import</span><span class="w"> </span><span class="nn">os</span>
367
+ <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="s2">&quot;TEST_EXECUTION.log&quot;</span><span class="p">,</span> <span class="s2">&quot;a&quot;</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
368
+ <span class="n">f</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;archive_node CALLED at </span><span class="si">{</span><span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span><span class="si">}</span><span class="se">\n</span><span class="s2">&quot;</span><span class="p">)</span>
369
+
370
+ <span class="n">node_runner</span> <span class="o">=</span> <span class="n">kwargs</span><span class="p">[</span><span class="s2">&quot;node_runner&quot;</span><span class="p">]</span>
371
+ <span class="n">db</span> <span class="o">=</span> <span class="n">context</span><span class="o">.</span><span class="n">db</span>
372
+ <span class="n">my_resource</span> <span class="o">=</span> <span class="n">context</span><span class="o">.</span><span class="n">config</span><span class="o">.</span><span class="n">resource</span>
373
+
374
+ <span class="c1"># Discover FileStacks</span>
375
+ <span class="n">all_file_stacks</span> <span class="o">=</span> <span class="p">[]</span>
376
+ <span class="kn">import</span><span class="w"> </span><span class="nn">sys</span>
377
+ <span class="n">sys</span><span class="o">.</span><span class="n">stderr</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;DEBUG: Discovery starting for resource </span><span class="si">{</span><span class="n">my_resource</span><span class="si">}</span><span class="se">\n</span><span class="s2">&quot;</span><span class="p">)</span>
378
+ <span class="n">sys</span><span class="o">.</span><span class="n">stderr</span><span class="o">.</span><span class="n">flush</span><span class="p">()</span>
379
+ <span class="k">if</span> <span class="n">archive_config</span><span class="o">.</span><span class="n">call_paths</span><span class="p">:</span>
380
+ <span class="kn">from</span><span class="w"> </span><span class="nn">simstack.models</span><span class="w"> </span><span class="kn">import</span> <span class="n">NodeRegistry</span><span class="p">,</span> <span class="n">FileListModel</span>
381
+
382
+ <span class="c1"># Find matching NodeRegistry entries</span>
383
+ <span class="n">node_query</span> <span class="o">=</span> <span class="p">{</span>
384
+ <span class="s2">&quot;call_path&quot;</span><span class="p">:</span> <span class="p">{</span><span class="s2">&quot;$in&quot;</span><span class="p">:</span> <span class="n">archive_config</span><span class="o">.</span><span class="n">call_paths</span><span class="p">},</span>
385
+ <span class="s2">&quot;parameters.resource.value&quot;</span><span class="p">:</span> <span class="n">my_resource</span>
386
+ <span class="p">}</span>
387
+ <span class="n">matching_nodes</span> <span class="o">=</span> <span class="k">await</span> <span class="n">db</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="n">NodeRegistry</span><span class="p">,</span> <span class="n">node_query</span><span class="p">)</span>
388
+
389
+ <span class="c1"># We also need mappings to know which collection to look into</span>
390
+ <span class="c1"># context.model_mappings has by_mapping and by_name</span>
391
+ <span class="n">filestack_mapping</span> <span class="o">=</span> <span class="n">context</span><span class="o">.</span><span class="n">model_mappings</span><span class="o">.</span><span class="n">get_by_name</span><span class="p">(</span><span class="s2">&quot;FileStack&quot;</span><span class="p">)</span>
392
+ <span class="n">filelist_mapping</span> <span class="o">=</span> <span class="n">context</span><span class="o">.</span><span class="n">model_mappings</span><span class="o">.</span><span class="n">get_by_name</span><span class="p">(</span><span class="s2">&quot;FileList&quot;</span><span class="p">)</span>
393
+ <span class="n">filelistmodel_mapping</span> <span class="o">=</span> <span class="n">context</span><span class="o">.</span><span class="n">model_mappings</span><span class="o">.</span><span class="n">get_by_name</span><span class="p">(</span><span class="s2">&quot;FileListModel&quot;</span><span class="p">)</span>
394
+
395
+ <span class="n">fs_mapping_str</span> <span class="o">=</span> <span class="n">filestack_mapping</span><span class="o">.</span><span class="n">mapping</span> <span class="k">if</span> <span class="n">filestack_mapping</span> <span class="k">else</span> <span class="s2">&quot;&quot;</span>
396
+ <span class="n">fl_mapping_str</span> <span class="o">=</span> <span class="n">filelist_mapping</span><span class="o">.</span><span class="n">mapping</span> <span class="k">if</span> <span class="n">filelist_mapping</span> <span class="k">else</span> <span class="s2">&quot;&quot;</span>
397
+ <span class="n">flm_mapping_str</span> <span class="o">=</span> <span class="n">filelistmodel_mapping</span><span class="o">.</span><span class="n">mapping</span> <span class="k">if</span> <span class="n">filelistmodel_mapping</span> <span class="k">else</span> <span class="s2">&quot;&quot;</span>
398
+
399
+ <span class="n">found_fs_ids</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
400
+ <span class="n">found_fl_ids</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
401
+ <span class="n">found_flm_ids</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
402
+
403
+ <span class="k">for</span> <span class="n">node</span> <span class="ow">in</span> <span class="n">matching_nodes</span><span class="p">:</span>
404
+ <span class="c1"># Check info_files</span>
405
+ <span class="k">for</span> <span class="n">fs</span> <span class="ow">in</span> <span class="n">node</span><span class="o">.</span><span class="n">info_files</span><span class="p">:</span>
406
+ <span class="n">found_fs_ids</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">fs</span><span class="o">.</span><span class="n">id</span><span class="p">)</span>
407
+
408
+ <span class="c1"># Check input_references and results_references</span>
409
+ <span class="k">for</span> <span class="n">ref</span> <span class="ow">in</span> <span class="n">node</span><span class="o">.</span><span class="n">input_references</span> <span class="o">+</span> <span class="n">node</span><span class="o">.</span><span class="n">results_references</span><span class="p">:</span>
410
+ <span class="k">if</span> <span class="n">ref</span><span class="o">.</span><span class="n">variable_mapping</span> <span class="o">==</span> <span class="n">fs_mapping_str</span><span class="p">:</span>
411
+ <span class="n">found_fs_ids</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">ref</span><span class="o">.</span><span class="n">reference</span><span class="p">)</span>
412
+ <span class="k">elif</span> <span class="n">ref</span><span class="o">.</span><span class="n">variable_mapping</span> <span class="o">==</span> <span class="n">fl_mapping_str</span><span class="p">:</span>
413
+ <span class="n">found_fl_ids</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">ref</span><span class="o">.</span><span class="n">reference</span><span class="p">)</span>
414
+ <span class="k">elif</span> <span class="n">ref</span><span class="o">.</span><span class="n">variable_mapping</span> <span class="o">==</span> <span class="n">flm_mapping_str</span><span class="p">:</span>
415
+ <span class="n">found_flm_ids</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">ref</span><span class="o">.</span><span class="n">reference</span><span class="p">)</span>
416
+
417
+ <span class="c1"># Retrieve FileStacks from found IDs</span>
418
+ <span class="k">if</span> <span class="n">found_fs_ids</span><span class="p">:</span>
419
+ <span class="n">file_stacks</span> <span class="o">=</span> <span class="k">await</span> <span class="n">db</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="n">FileStack</span><span class="p">,</span> <span class="p">{</span><span class="s2">&quot;_id&quot;</span><span class="p">:</span> <span class="p">{</span><span class="s2">&quot;$in&quot;</span><span class="p">:</span> <span class="nb">list</span><span class="p">(</span><span class="n">found_fs_ids</span><span class="p">)}})</span>
420
+ <span class="n">all_file_stacks</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="n">file_stacks</span><span class="p">)</span>
421
+
422
+ <span class="c1"># Retrieve FileLists and their FileStacks</span>
423
+ <span class="k">if</span> <span class="n">found_flm_ids</span><span class="p">:</span>
424
+ <span class="n">file_list_models</span> <span class="o">=</span> <span class="k">await</span> <span class="n">db</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="n">FileListModel</span><span class="p">,</span> <span class="p">{</span><span class="s2">&quot;_id&quot;</span><span class="p">:</span> <span class="p">{</span><span class="s2">&quot;$in&quot;</span><span class="p">:</span> <span class="nb">list</span><span class="p">(</span><span class="n">found_flm_ids</span><span class="p">)}})</span>
425
+ <span class="k">for</span> <span class="n">flm</span> <span class="ow">in</span> <span class="n">file_list_models</span><span class="p">:</span>
426
+ <span class="c1"># FileListModel uses ObjectListMixin[FileStack], elements are ObjectIds</span>
427
+ <span class="c1"># We need to fetch the actual FileStack objects.</span>
428
+ <span class="k">if</span> <span class="n">flm</span><span class="o">.</span><span class="n">elements</span><span class="p">:</span>
429
+ <span class="n">stacks</span> <span class="o">=</span> <span class="k">await</span> <span class="n">db</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="n">FileStack</span><span class="p">,</span> <span class="p">{</span><span class="s2">&quot;_id&quot;</span><span class="p">:</span> <span class="p">{</span><span class="s2">&quot;$in&quot;</span><span class="p">:</span> <span class="n">flm</span><span class="o">.</span><span class="n">elements</span><span class="p">}})</span>
430
+ <span class="n">all_file_stacks</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="n">stacks</span><span class="p">)</span>
431
+
432
+ <span class="c1"># Deduplicate all_file_stacks by ID</span>
433
+ <span class="n">unique_stacks</span> <span class="o">=</span> <span class="p">{}</span>
434
+ <span class="k">for</span> <span class="n">fs</span> <span class="ow">in</span> <span class="n">all_file_stacks</span><span class="p">:</span>
435
+ <span class="n">unique_stacks</span><span class="p">[</span><span class="n">fs</span><span class="o">.</span><span class="n">id</span><span class="p">]</span> <span class="o">=</span> <span class="n">fs</span>
436
+ <span class="n">all_file_stacks</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="n">unique_stacks</span><span class="o">.</span><span class="n">values</span><span class="p">())</span>
437
+
438
+ <span class="k">else</span><span class="p">:</span>
439
+ <span class="c1"># Existing logic: Query FileStack collection directly</span>
440
+ <span class="n">query</span> <span class="o">=</span> <span class="p">{}</span>
441
+ <span class="k">if</span> <span class="n">archive_config</span><span class="o">.</span><span class="n">min_size</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
442
+ <span class="n">query</span><span class="p">[</span><span class="s2">&quot;size&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span><span class="s2">&quot;$gte&quot;</span><span class="p">:</span> <span class="n">archive_config</span><span class="o">.</span><span class="n">min_size</span><span class="p">}</span>
443
+
444
+ <span class="k">if</span> <span class="n">archive_config</span><span class="o">.</span><span class="n">filter_by_resource</span><span class="p">:</span>
445
+ <span class="n">query</span><span class="p">[</span><span class="s2">&quot;locations.resource.value&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">my_resource</span>
446
+
447
+ <span class="n">all_file_stacks</span> <span class="o">=</span> <span class="k">await</span> <span class="n">db</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="n">FileStack</span><span class="p">,</span> <span class="n">query</span><span class="p">)</span>
448
+
449
+ <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;DEBUG: Found </span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="n">all_file_stacks</span><span class="p">)</span><span class="si">}</span><span class="s2"> FileStacks&quot;</span><span class="p">)</span>
450
+ <span class="n">filtered_file_stacks</span> <span class="o">=</span> <span class="p">[]</span>
451
+
452
+ <span class="k">for</span> <span class="n">fs</span> <span class="ow">in</span> <span class="n">all_file_stacks</span><span class="p">:</span>
453
+ <span class="c1"># Filter by date if specified</span>
454
+ <span class="c1"># Since FileStack doesn&#39;t have a date, we check its locations.</span>
455
+ <span class="k">if</span> <span class="n">archive_config</span><span class="o">.</span><span class="n">start_date</span> <span class="ow">or</span> <span class="n">archive_config</span><span class="o">.</span><span class="n">end_date</span><span class="p">:</span>
456
+ <span class="n">valid_date</span> <span class="o">=</span> <span class="kc">False</span>
457
+ <span class="k">for</span> <span class="n">loc</span> <span class="ow">in</span> <span class="n">fs</span><span class="o">.</span><span class="n">locations</span><span class="p">:</span>
458
+ <span class="k">if</span> <span class="n">loc</span><span class="o">.</span><span class="n">created_at</span><span class="p">:</span>
459
+ <span class="k">if</span> <span class="n">archive_config</span><span class="o">.</span><span class="n">start_date</span> <span class="ow">and</span> <span class="n">loc</span><span class="o">.</span><span class="n">created_at</span> <span class="o">&lt;</span> <span class="n">archive_config</span><span class="o">.</span><span class="n">start_date</span><span class="p">:</span>
460
+ <span class="k">continue</span>
461
+ <span class="k">if</span> <span class="n">archive_config</span><span class="o">.</span><span class="n">end_date</span> <span class="ow">and</span> <span class="n">loc</span><span class="o">.</span><span class="n">created_at</span> <span class="o">&gt;</span> <span class="n">archive_config</span><span class="o">.</span><span class="n">end_date</span><span class="p">:</span>
462
+ <span class="k">continue</span>
463
+ <span class="n">valid_date</span> <span class="o">=</span> <span class="kc">True</span>
464
+ <span class="k">break</span>
465
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">valid_date</span><span class="p">:</span>
466
+ <span class="k">continue</span>
467
+
468
+ <span class="c1"># Filter by patterns</span>
469
+ <span class="k">if</span> <span class="n">archive_config</span><span class="o">.</span><span class="n">include_patterns</span><span class="p">:</span>
470
+ <span class="kn">import</span><span class="w"> </span><span class="nn">fnmatch</span>
471
+ <span class="k">if</span> <span class="ow">not</span> <span class="nb">any</span><span class="p">(</span><span class="n">fnmatch</span><span class="o">.</span><span class="n">fnmatch</span><span class="p">(</span><span class="n">fs</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="n">pat</span><span class="p">)</span> <span class="k">for</span> <span class="n">pat</span> <span class="ow">in</span> <span class="n">archive_config</span><span class="o">.</span><span class="n">include_patterns</span><span class="p">):</span>
472
+ <span class="k">continue</span>
473
+
474
+ <span class="k">if</span> <span class="n">archive_config</span><span class="o">.</span><span class="n">exclude_patterns</span><span class="p">:</span>
475
+ <span class="kn">import</span><span class="w"> </span><span class="nn">fnmatch</span>
476
+ <span class="k">if</span> <span class="nb">any</span><span class="p">(</span><span class="n">fnmatch</span><span class="o">.</span><span class="n">fnmatch</span><span class="p">(</span><span class="n">fs</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="n">pat</span><span class="p">)</span> <span class="k">for</span> <span class="n">pat</span> <span class="ow">in</span> <span class="n">archive_config</span><span class="o">.</span><span class="n">exclude_patterns</span><span class="p">):</span>
477
+ <span class="k">continue</span>
478
+
479
+ <span class="n">filtered_file_stacks</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">fs</span><span class="p">)</span>
480
+
481
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">filtered_file_stacks</span><span class="p">:</span>
482
+ <span class="n">node_runner</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;No files found matching the archival criteria.&quot;</span><span class="p">)</span>
483
+ <span class="k">return</span> <span class="n">BooleanDataList</span><span class="p">(</span><span class="n">field_name</span><span class="o">=</span><span class="s2">&quot;archive_results&quot;</span><span class="p">)</span>
484
+
485
+ <span class="c1"># 2. Add them to FileList</span>
486
+ <span class="n">archive_file_list</span> <span class="o">=</span> <span class="n">FileList</span><span class="p">(</span><span class="n">field_name</span><span class="o">=</span><span class="s2">&quot;archive_files&quot;</span><span class="p">)</span>
487
+ <span class="k">for</span> <span class="n">fs</span> <span class="ow">in</span> <span class="n">filtered_file_stacks</span><span class="p">:</span>
488
+ <span class="n">archive_file_list</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">fs</span><span class="p">)</span>
489
+
490
+ <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;DEBUG: archive_file_list has </span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="n">archive_file_list</span><span class="p">)</span><span class="si">}</span><span class="s2"> files&quot;</span><span class="p">)</span>
491
+ <span class="c1"># 3. Call archive_files on the filelist</span>
492
+ <span class="c1"># Note: archive_files is a @node, we can call it directly.</span>
493
+ <span class="c1"># It returns a BooleanDataList</span>
494
+
495
+ <span class="c1"># To handle mocks and different return types, we ensure we get a BooleanDataList</span>
496
+ <span class="c1"># When calling a @node decorated function, it returns a SimstackResult or Model in sync mode,</span>
497
+ <span class="c1"># or the result of the function if it was run locally.</span>
498
+ <span class="c1"># If we are in the same process and calling it, it might be running via NodeRunner.</span>
499
+ <span class="n">raw_results</span> <span class="o">=</span> <span class="k">await</span> <span class="n">archive_files</span><span class="p">(</span><span class="n">archive_file_list</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
500
+ <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;DEBUG: raw_results type=</span><span class="si">{</span><span class="nb">type</span><span class="p">(</span><span class="n">raw_results</span><span class="p">)</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
501
+ <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">raw_results</span><span class="p">,</span> <span class="n">BooleanDataList</span><span class="p">):</span>
502
+ <span class="n">archive_results</span> <span class="o">=</span> <span class="n">raw_results</span>
503
+ <span class="k">else</span><span class="p">:</span>
504
+ <span class="c1"># If mocked or returned as list/iterable</span>
505
+ <span class="n">archive_results</span> <span class="o">=</span> <span class="n">BooleanDataList</span><span class="p">(</span><span class="n">field_name</span><span class="o">=</span><span class="s2">&quot;archive_results&quot;</span><span class="p">)</span>
506
+ <span class="kn">from</span><span class="w"> </span><span class="nn">simstack.models</span><span class="w"> </span><span class="kn">import</span> <span class="n">BooleanData</span>
507
+ <span class="k">if</span> <span class="n">raw_results</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
508
+ <span class="c1"># Handle possible async result if not awaited properly by decorator</span>
509
+ <span class="k">if</span> <span class="n">inspect</span><span class="o">.</span><span class="n">isawaitable</span><span class="p">(</span><span class="n">raw_results</span><span class="p">):</span>
510
+ <span class="n">raw_results</span> <span class="o">=</span> <span class="k">await</span> <span class="n">raw_results</span>
511
+
512
+ <span class="k">for</span> <span class="n">r</span> <span class="ow">in</span> <span class="n">raw_results</span><span class="p">:</span>
513
+ <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">r</span><span class="p">,</span> <span class="nb">bool</span><span class="p">):</span>
514
+ <span class="n">archive_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">BooleanData</span><span class="p">(</span><span class="n">value</span><span class="o">=</span><span class="n">r</span><span class="p">))</span>
515
+ <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">r</span><span class="p">,</span> <span class="n">BooleanData</span><span class="p">):</span>
516
+ <span class="n">archive_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">r</span><span class="p">)</span>
517
+ <span class="k">else</span><span class="p">:</span>
518
+ <span class="c1"># try to save it if it&#39;s a model</span>
519
+ <span class="n">archive_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">r</span><span class="p">)</span>
520
+
521
+ <span class="c1"># 4. Delete local FileInstances for successfull archival</span>
522
+ <span class="c1"># archive_results is a BooleanDataList, elements are BooleanData (presumably)</span>
523
+ <span class="c1"># Wait, archive_files returns BooleanDataList. Let&#39;s check BooleanDataList implementation.</span>
524
+
525
+ <span class="c1"># Actually archive_files returns BooleanDataList where each element corresponds to a FileStack in archive_file_list</span>
526
+ <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">success_data</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">archive_results</span><span class="p">):</span>
527
+ <span class="k">if</span> <span class="n">success_data</span><span class="o">.</span><span class="n">value</span><span class="p">:</span>
528
+ <span class="n">fs</span> <span class="o">=</span> <span class="n">archive_file_list</span><span class="p">[</span><span class="n">i</span><span class="p">]</span>
529
+ <span class="c1"># Delete local FileInstances</span>
530
+ <span class="c1"># We need to know which ones are &quot;local&quot;.</span>
531
+ <span class="c1"># Usually those are on the current resource.</span>
532
+ <span class="n">current_resource</span> <span class="o">=</span> <span class="n">context</span><span class="o">.</span><span class="n">config</span><span class="o">.</span><span class="n">resource</span>
533
+
534
+ <span class="n">remaining_locations</span> <span class="o">=</span> <span class="p">[]</span>
535
+ <span class="k">for</span> <span class="n">loc</span> <span class="ow">in</span> <span class="n">fs</span><span class="o">.</span><span class="n">locations</span><span class="p">:</span>
536
+ <span class="c1"># If it&#39;s on the current resource and is a local path, delete it</span>
537
+ <span class="k">if</span> <span class="n">loc</span><span class="o">.</span><span class="n">resource</span><span class="o">.</span><span class="n">value</span> <span class="o">==</span> <span class="n">current_resource</span> <span class="ow">and</span> <span class="n">loc</span><span class="o">.</span><span class="n">location_type</span> <span class="o">==</span> <span class="s2">&quot;local_path&quot;</span><span class="p">:</span>
538
+ <span class="k">try</span><span class="p">:</span>
539
+ <span class="n">local_path</span> <span class="o">=</span> <span class="n">Path</span><span class="p">(</span><span class="n">context</span><span class="o">.</span><span class="n">config</span><span class="o">.</span><span class="n">workdir</span><span class="p">)</span> <span class="o">/</span> <span class="n">loc</span><span class="o">.</span><span class="n">path</span>
540
+ <span class="k">if</span> <span class="n">local_path</span><span class="o">.</span><span class="n">exists</span><span class="p">():</span>
541
+ <span class="k">if</span> <span class="n">local_path</span><span class="o">.</span><span class="n">is_file</span><span class="p">():</span>
542
+ <span class="n">local_path</span><span class="o">.</span><span class="n">unlink</span><span class="p">()</span>
543
+ <span class="k">elif</span> <span class="n">local_path</span><span class="o">.</span><span class="n">is_dir</span><span class="p">():</span>
544
+ <span class="n">shutil</span><span class="o">.</span><span class="n">rmtree</span><span class="p">(</span><span class="n">local_path</span><span class="p">)</span>
545
+ <span class="n">node_runner</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Deleted local instance of </span><span class="si">{</span><span class="n">fs</span><span class="o">.</span><span class="n">name</span><span class="si">}</span><span class="s2"> at </span><span class="si">{</span><span class="n">local_path</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
546
+ <span class="k">else</span><span class="p">:</span>
547
+ <span class="n">node_runner</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Local path </span><span class="si">{</span><span class="n">local_path</span><span class="si">}</span><span class="s2"> for </span><span class="si">{</span><span class="n">fs</span><span class="o">.</span><span class="n">name</span><span class="si">}</span><span class="s2"> does not exist&quot;</span><span class="p">)</span>
548
+ <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
549
+ <span class="n">node_runner</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Failed to delete local instance of </span><span class="si">{</span><span class="n">fs</span><span class="o">.</span><span class="n">name</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
550
+ <span class="n">remaining_locations</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">loc</span><span class="p">)</span>
551
+ <span class="k">else</span><span class="p">:</span>
552
+ <span class="n">remaining_locations</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">loc</span><span class="p">)</span>
553
+
554
+ <span class="n">fs</span><span class="o">.</span><span class="n">locations</span> <span class="o">=</span> <span class="n">remaining_locations</span>
555
+ <span class="k">await</span> <span class="n">db</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="n">fs</span><span class="p">)</span>
556
+
557
+ <span class="k">return</span> <span class="n">archive_results</span></div>
558
+
559
+
560
+ </pre></div>
561
+ </article>
562
+ </div>
563
+ <footer>
564
+
565
+ <div class="related-pages">
566
+
567
+
568
+ </div>
569
+ <div class="bottom-of-page">
570
+ <div class="left-details">
571
+ <div class="copyright">
572
+ Copyright &#169; 2026, Wolfgang Wenzel
573
+ </div>
574
+ Made with <a href="https://www.sphinx-doc.org/">Sphinx</a> and <a class="muted-link" href="https://pradyunsg.me">@pradyunsg</a>'s
575
+
576
+ <a href="https://github.com/pradyunsg/furo">Furo</a>
577
+
578
+ </div>
579
+ <div class="right-details">
580
+
581
+ </div>
582
+ </div>
583
+
584
+ </footer>
585
+ </div>
586
+ <aside class="toc-drawer no-toc">
587
+
588
+
589
+
590
+ </aside>
591
+ </div>
592
+ </div><script src="../../../_static/documentation_options.js?v=5929fcd5"></script>
593
+ <script src="../../../_static/doctools.js?v=fd6eb6e6"></script>
594
+ <script src="../../../_static/sphinx_highlight.js?v=6ffebe34"></script>
595
+ <script src="../../../_static/scripts/furo.js?v=46bd48cc"></script>
596
+ <script data-docs-built-at="2026-07-17 10:50:10 UTC" src="../../../_static/docs-build-badge.js?v=57cb9fc6"></script>
597
+ </body>
598
+ </html>