simstack 2.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (354) hide show
  1. simstack/__init__.py +0 -0
  2. simstack/_version.py +24 -0
  3. simstack/core/__init__.py +0 -0
  4. simstack/core/artifacts.py +406 -0
  5. simstack/core/asnyc_helper.py +14 -0
  6. simstack/core/config_file.py +60 -0
  7. simstack/core/context.py +283 -0
  8. simstack/core/definitions.py +22 -0
  9. simstack/core/engine.py +115 -0
  10. simstack/core/find_simstack_modules.py +41 -0
  11. simstack/core/hash.py +179 -0
  12. simstack/core/node.py +1137 -0
  13. simstack/core/node_claim.py +31 -0
  14. simstack/core/node_runner.py +362 -0
  15. simstack/core/process_results.py +102 -0
  16. simstack/core/recompute_artifacts.py +117 -0
  17. simstack/core/reset_db.py +12 -0
  18. simstack/core/resource_assignment.py +222 -0
  19. simstack/core/resources.py +90 -0
  20. simstack/core/route_table.py +24 -0
  21. simstack/core/run_docker.py +109 -0
  22. simstack/core/run_node.py +71 -0
  23. simstack/core/runner.py +169 -0
  24. simstack/core/services/__init__.py +11 -0
  25. simstack/core/services/base_service.py +191 -0
  26. simstack/core/services/file_transfer_service.py +117 -0
  27. simstack/core/services/git_restart_service.py +12 -0
  28. simstack/core/services/git_uv_update_service.py +105 -0
  29. simstack/core/services/node_execution_service.py +182 -0
  30. simstack/core/services/resource_branch_monitor_service.py +71 -0
  31. simstack/core/services/runner_cleanup_service.py +149 -0
  32. simstack/core/services/runner_manager.py +163 -0
  33. simstack/core/services/runner_status_service.py +10 -0
  34. simstack/core/services/slurm_status_service.py +67 -0
  35. simstack/core/services/stop_check_service.py +27 -0
  36. simstack/core/services/timeout_restart_service.py +26 -0
  37. simstack/core/simstack_result.py +39 -0
  38. simstack/core/submit_node.py +217 -0
  39. simstack/core/task_id.py +22 -0
  40. simstack/methods/__init__.py +0 -0
  41. simstack/methods/archive_files.py +292 -0
  42. simstack/methods/fire_and_forget_runner.py +113 -0
  43. simstack/methods/generate_test.py +125 -0
  44. simstack/methods/get_file.py +26 -0
  45. simstack/methods/mass_runner.py +154 -0
  46. simstack/methods/multiple_executor.py +307 -0
  47. simstack/methods/switch_git.py +16 -0
  48. simstack/methods/upload_helpers.py +81 -0
  49. simstack/models/__init__.py +74 -0
  50. simstack/models/array_list.py +21 -0
  51. simstack/models/array_storage.py +83 -0
  52. simstack/models/artifact_models.py +127 -0
  53. simstack/models/base_lists.py +51 -0
  54. simstack/models/base_types.py +209 -0
  55. simstack/models/charts_artifact.py +690 -0
  56. simstack/models/dataset.py +451 -0
  57. simstack/models/dataset_metadata.py +390 -0
  58. simstack/models/datasettuple.py +587 -0
  59. simstack/models/file_instance.py +150 -0
  60. simstack/models/file_list.py +45 -0
  61. simstack/models/files.py +424 -0
  62. simstack/models/fire_and_forget_result.py +19 -0
  63. simstack/models/images2d.py +113 -0
  64. simstack/models/log_entry_model.py +37 -0
  65. simstack/models/models.py +52 -0
  66. simstack/models/named_data_reference.py +36 -0
  67. simstack/models/node_registry.py +96 -0
  68. simstack/models/pandas_model.py +239 -0
  69. simstack/models/parameters.py +402 -0
  70. simstack/models/pickle_models.py +300 -0
  71. simstack/models/project.py +13 -0
  72. simstack/models/resource_assignment.py +170 -0
  73. simstack/models/resource_definition.py +125 -0
  74. simstack/models/runner_model.py +39 -0
  75. simstack/models/simple_table.py +79 -0
  76. simstack/models/simstack_model.py +97 -0
  77. simstack/models/slurm_info.py +17 -0
  78. simstack/models/table_artifact.py +87 -0
  79. simstack/models/tag.py +12 -0
  80. simstack/models/unused_test_file.py +5 -0
  81. simstack/simstack_marker.com +1 -0
  82. simstack/static/docs/.buildinfo +4 -0
  83. simstack/static/docs/_images/GUI Concept.webp +0 -0
  84. simstack/static/docs/_modules/index.html +434 -0
  85. simstack/static/docs/_modules/simstack/core/artifacts.html +743 -0
  86. simstack/static/docs/_modules/simstack/core/asnyc_helper.html +330 -0
  87. simstack/static/docs/_modules/simstack/core/config_file.html +379 -0
  88. simstack/static/docs/_modules/simstack/core/definitions.html +341 -0
  89. simstack/static/docs/_modules/simstack/core/engine.html +437 -0
  90. simstack/static/docs/_modules/simstack/core/find_simstack_modules.html +360 -0
  91. simstack/static/docs/_modules/simstack/core/hash.html +528 -0
  92. simstack/static/docs/_modules/simstack/core/node.html +1392 -0
  93. simstack/static/docs/_modules/simstack/core/node_claim.html +347 -0
  94. simstack/static/docs/_modules/simstack/core/node_runner.html +711 -0
  95. simstack/static/docs/_modules/simstack/core/process_results.html +418 -0
  96. simstack/static/docs/_modules/simstack/core/recompute_artifacts.html +433 -0
  97. simstack/static/docs/_modules/simstack/core/reset_db.html +328 -0
  98. simstack/static/docs/_modules/simstack/core/resource_assignment.html +553 -0
  99. simstack/static/docs/_modules/simstack/core/resources.html +424 -0
  100. simstack/static/docs/_modules/simstack/core/route_table.html +349 -0
  101. simstack/static/docs/_modules/simstack/core/run_docker.html +425 -0
  102. simstack/static/docs/_modules/simstack/core/run_node.html +390 -0
  103. simstack/static/docs/_modules/simstack/core/runner.html +491 -0
  104. simstack/static/docs/_modules/simstack/core/services/base_service.html +522 -0
  105. simstack/static/docs/_modules/simstack/core/services/file_transfer_service.html +436 -0
  106. simstack/static/docs/_modules/simstack/core/services/git_restart_service.html +331 -0
  107. simstack/static/docs/_modules/simstack/core/services/git_uv_update_service.html +427 -0
  108. simstack/static/docs/_modules/simstack/core/services/node_execution_service.html +507 -0
  109. simstack/static/docs/_modules/simstack/core/services/resource_branch_monitor_service.html +390 -0
  110. simstack/static/docs/_modules/simstack/core/services/runner_cleanup_service.html +468 -0
  111. simstack/static/docs/_modules/simstack/core/services/runner_manager.html +485 -0
  112. simstack/static/docs/_modules/simstack/core/services/runner_status_service.html +329 -0
  113. simstack/static/docs/_modules/simstack/core/services/slurm_status_service.html +386 -0
  114. simstack/static/docs/_modules/simstack/core/services/stop_check_service.html +346 -0
  115. simstack/static/docs/_modules/simstack/core/services/timeout_restart_service.html +345 -0
  116. simstack/static/docs/_modules/simstack/core/simstack_result.html +355 -0
  117. simstack/static/docs/_modules/simstack/core/submit_node.html +536 -0
  118. simstack/static/docs/_modules/simstack/core/task_id.html +344 -0
  119. simstack/static/docs/_modules/simstack/methods/archive_files.html +598 -0
  120. simstack/static/docs/_modules/simstack/methods/fire_and_forget_runner.html +432 -0
  121. simstack/static/docs/_modules/simstack/methods/generate_test.html +443 -0
  122. simstack/static/docs/_modules/simstack/methods/get_file.html +342 -0
  123. simstack/static/docs/_modules/simstack/methods/mass_runner.html +476 -0
  124. simstack/static/docs/_modules/simstack/methods/multiple_executor.html +674 -0
  125. simstack/static/docs/_modules/simstack/methods/switch_git.html +332 -0
  126. simstack/static/docs/_modules/simstack/methods/upload_helpers.html +403 -0
  127. simstack/static/docs/_modules/simstack/models/array_list.html +337 -0
  128. simstack/static/docs/_modules/simstack/models/array_storage.html +417 -0
  129. simstack/static/docs/_modules/simstack/models/base_lists.html +374 -0
  130. simstack/static/docs/_modules/simstack/models/base_types.html +588 -0
  131. simstack/static/docs/_modules/simstack/models/charts_artifact.html +1035 -0
  132. simstack/static/docs/_modules/simstack/models/dataset.html +866 -0
  133. simstack/static/docs/_modules/simstack/models/dataset_metadata.html +711 -0
  134. simstack/static/docs/_modules/simstack/models/datasettuple.html +1005 -0
  135. simstack/static/docs/_modules/simstack/models/file_instance.html +472 -0
  136. simstack/static/docs/_modules/simstack/models/file_list.html +367 -0
  137. simstack/static/docs/_modules/simstack/models/files.html +773 -0
  138. simstack/static/docs/_modules/simstack/models/fire_and_forget_result.html +335 -0
  139. simstack/static/docs/_modules/simstack/models/images2d.html +447 -0
  140. simstack/static/docs/_modules/simstack/models/log_entry_model.html +356 -0
  141. simstack/static/docs/_modules/simstack/models/models.html +374 -0
  142. simstack/static/docs/_modules/simstack/models/named_data_reference.html +355 -0
  143. simstack/static/docs/_modules/simstack/models/node_registry.html +415 -0
  144. simstack/static/docs/_modules/simstack/models/pandas_model.html +570 -0
  145. simstack/static/docs/_modules/simstack/models/parameters.html +739 -0
  146. simstack/static/docs/_modules/simstack/models/pickle_models.html +631 -0
  147. simstack/static/docs/_modules/simstack/models/project.html +329 -0
  148. simstack/static/docs/_modules/simstack/models/resource_assignment.html +501 -0
  149. simstack/static/docs/_modules/simstack/models/resource_definition.html +477 -0
  150. simstack/static/docs/_modules/simstack/models/runner_model.html +361 -0
  151. simstack/static/docs/_modules/simstack/models/simple_table.html +413 -0
  152. simstack/static/docs/_modules/simstack/models/simstack_model.html +416 -0
  153. simstack/static/docs/_modules/simstack/models/slurm_info.html +333 -0
  154. simstack/static/docs/_modules/simstack/models/table_artifact.html +406 -0
  155. simstack/static/docs/_modules/simstack/models/tag.html +328 -0
  156. simstack/static/docs/_modules/simstack/tables/model_table.html +519 -0
  157. simstack/static/docs/_modules/simstack/tables/node_children.html +538 -0
  158. simstack/static/docs/_modules/simstack/tables/node_table.html +854 -0
  159. simstack/static/docs/_modules/simstack/tables/table_builder.html +642 -0
  160. simstack/static/docs/_modules/simstack/util/async_zip_utils.html +337 -0
  161. simstack/static/docs/_modules/simstack/util/b64mixin.html +339 -0
  162. simstack/static/docs/_modules/simstack/util/cleaned_json_schema.html +336 -0
  163. simstack/static/docs/_modules/simstack/util/config_reader.html +498 -0
  164. simstack/static/docs/_modules/simstack/util/custom_model_dump.html +391 -0
  165. simstack/static/docs/_modules/simstack/util/database_information.html +488 -0
  166. simstack/static/docs/_modules/simstack/util/db.html +1392 -0
  167. simstack/static/docs/_modules/simstack/util/db_logger.html +448 -0
  168. simstack/static/docs/_modules/simstack/util/default_from_dict.html +376 -0
  169. simstack/static/docs/_modules/simstack/util/directory_iterator.html +553 -0
  170. simstack/static/docs/_modules/simstack/util/docstring_parser.html +526 -0
  171. simstack/static/docs/_modules/simstack/util/file_hashing.html +394 -0
  172. simstack/static/docs/_modules/simstack/util/file_transfer_client.html +791 -0
  173. simstack/static/docs/_modules/simstack/util/files.html +733 -0
  174. simstack/static/docs/_modules/simstack/util/find_python_files.html +349 -0
  175. simstack/static/docs/_modules/simstack/util/generate_ui_schema.html +396 -0
  176. simstack/static/docs/_modules/simstack/util/generic_list_mixin.html +487 -0
  177. simstack/static/docs/_modules/simstack/util/get_module_path.html +336 -0
  178. simstack/static/docs/_modules/simstack/util/git_pull.html +381 -0
  179. simstack/static/docs/_modules/simstack/util/git_repository_status.html +411 -0
  180. simstack/static/docs/_modules/simstack/util/import_module.html +377 -0
  181. simstack/static/docs/_modules/simstack/util/importer.html +637 -0
  182. simstack/static/docs/_modules/simstack/util/init_data_source.html +357 -0
  183. simstack/static/docs/_modules/simstack/util/make_graph.html +423 -0
  184. simstack/static/docs/_modules/simstack/util/make_table.html +1171 -0
  185. simstack/static/docs/_modules/simstack/util/mappings.html +378 -0
  186. simstack/static/docs/_modules/simstack/util/minimal_route_finder.html +388 -0
  187. simstack/static/docs/_modules/simstack/util/mock_db.html +472 -0
  188. simstack/static/docs/_modules/simstack/util/module_path_checker.html +354 -0
  189. simstack/static/docs/_modules/simstack/util/object_list_mixin.html +649 -0
  190. simstack/static/docs/_modules/simstack/util/path_manager.html +566 -0
  191. simstack/static/docs/_modules/simstack/util/project_root_finder.html +357 -0
  192. simstack/static/docs/_modules/simstack/util/queue_watcher.html +593 -0
  193. simstack/static/docs/_modules/simstack/util/resource_config.html +532 -0
  194. simstack/static/docs/_modules/simstack/util/route_finder.html +390 -0
  195. simstack/static/docs/_modules/simstack/util/runner_util.html +487 -0
  196. simstack/static/docs/_modules/simstack/util/runner_utils.html +468 -0
  197. simstack/static/docs/_modules/simstack/util/safe_code_executor.html +510 -0
  198. simstack/static/docs/_modules/simstack/util/sample_config.html +414 -0
  199. simstack/static/docs/_modules/simstack/util/sensitive_content_filter.html +364 -0
  200. simstack/static/docs/_modules/simstack/util/setup_logging.html +365 -0
  201. simstack/static/docs/_modules/simstack/util/submit_to_watchdog.html +425 -0
  202. simstack/static/docs/_modules/simstack/util/toml_reader.html +482 -0
  203. simstack/static/docs/_modules/simstack/util/transform_file_name.html +357 -0
  204. simstack/static/docs/_modules/simstack/util/ui_tools.html +541 -0
  205. simstack/static/docs/_sources/architecture.rst.txt +25 -0
  206. simstack/static/docs/_sources/array_lists.rst.txt +60 -0
  207. simstack/static/docs/_sources/ci_pipeline.rst.txt +34 -0
  208. simstack/static/docs/_sources/data.rst.txt +175 -0
  209. simstack/static/docs/_sources/database_migrations.rst.txt +310 -0
  210. simstack/static/docs/_sources/datasets.rst.txt +93 -0
  211. simstack/static/docs/_sources/developer_guide.rst.txt +25 -0
  212. simstack/static/docs/_sources/docker_nodes.rst.txt +149 -0
  213. simstack/static/docs/_sources/files.rst.txt +159 -0
  214. simstack/static/docs/_sources/fire_and_forget_runner.rst.txt +67 -0
  215. simstack/static/docs/_sources/index.rst.txt +25 -0
  216. simstack/static/docs/_sources/installation.rst.txt +213 -0
  217. simstack/static/docs/_sources/lists.rst.txt +111 -0
  218. simstack/static/docs/_sources/mass_runner.rst.txt +91 -0
  219. simstack/static/docs/_sources/misc.rst.txt +106 -0
  220. simstack/static/docs/_sources/overview.rst.txt +91 -0
  221. simstack/static/docs/_sources/recovery.rst.txt +38 -0
  222. simstack/static/docs/_sources/resource_config.rst.txt +141 -0
  223. simstack/static/docs/_sources/simstack/modules/modules.rst.txt +7 -0
  224. simstack/static/docs/_sources/simstack/modules/simstack.core.rst.txt +205 -0
  225. simstack/static/docs/_sources/simstack/modules/simstack.core.services.rst.txt +109 -0
  226. simstack/static/docs/_sources/simstack/modules/simstack.methods.rst.txt +77 -0
  227. simstack/static/docs/_sources/simstack/modules/simstack.models.rst.txt +261 -0
  228. simstack/static/docs/_sources/simstack/modules/simstack.rst.txt +22 -0
  229. simstack/static/docs/_sources/simstack/modules/simstack.tables.rst.txt +45 -0
  230. simstack/static/docs/_sources/simstack/modules/simstack.util.rst.txt +381 -0
  231. simstack/static/docs/_sources/submit.rst.txt +141 -0
  232. simstack/static/docs/_sources/tables.rst.txt +50 -0
  233. simstack/static/docs/_sources/testing_complex_nodes.rst.txt +92 -0
  234. simstack/static/docs/_sources/ui_documentation.rst.txt +272 -0
  235. simstack/static/docs/_sources/workflows.rst.txt +446 -0
  236. simstack/static/docs/_static/base-stemmer.js +476 -0
  237. simstack/static/docs/_static/basic.css +906 -0
  238. simstack/static/docs/_static/cobalt2-theme.css +1 -0
  239. simstack/static/docs/_static/debug.css +69 -0
  240. simstack/static/docs/_static/docs-build-badge.css +51 -0
  241. simstack/static/docs/_static/docs-build-badge.js +51 -0
  242. simstack/static/docs/_static/doctools.js +150 -0
  243. simstack/static/docs/_static/documentation_options.js +13 -0
  244. simstack/static/docs/_static/english-stemmer.js +1066 -0
  245. simstack/static/docs/_static/file.png +0 -0
  246. simstack/static/docs/_static/language_data.js +13 -0
  247. simstack/static/docs/_static/minus.png +0 -0
  248. simstack/static/docs/_static/plus.png +0 -0
  249. simstack/static/docs/_static/pygments.css +250 -0
  250. simstack/static/docs/_static/scripts/furo-extensions.js +0 -0
  251. simstack/static/docs/_static/scripts/furo.js +3 -0
  252. simstack/static/docs/_static/scripts/furo.js.LICENSE.txt +7 -0
  253. simstack/static/docs/_static/scripts/furo.js.map +1 -0
  254. simstack/static/docs/_static/searchtools.js +693 -0
  255. simstack/static/docs/_static/skeleton.css +296 -0
  256. simstack/static/docs/_static/sphinx_highlight.js +159 -0
  257. simstack/static/docs/_static/styles/furo-extensions.css +2 -0
  258. simstack/static/docs/_static/styles/furo-extensions.css.map +1 -0
  259. simstack/static/docs/_static/styles/furo.css +2 -0
  260. simstack/static/docs/_static/styles/furo.css.map +1 -0
  261. simstack/static/docs/architecture.html +340 -0
  262. simstack/static/docs/array_lists.html +678 -0
  263. simstack/static/docs/ci_pipeline.html +385 -0
  264. simstack/static/docs/data.html +545 -0
  265. simstack/static/docs/database_migrations.html +652 -0
  266. simstack/static/docs/datasets.html +1268 -0
  267. simstack/static/docs/developer_guide.html +359 -0
  268. simstack/static/docs/docker_nodes.html +503 -0
  269. simstack/static/docs/files.html +718 -0
  270. simstack/static/docs/fire_and_forget_runner.html +641 -0
  271. simstack/static/docs/genindex.html +6390 -0
  272. simstack/static/docs/index.html +516 -0
  273. simstack/static/docs/installation.html +553 -0
  274. simstack/static/docs/lists.html +478 -0
  275. simstack/static/docs/mass_runner.html +481 -0
  276. simstack/static/docs/misc.html +454 -0
  277. simstack/static/docs/objects.inv +0 -0
  278. simstack/static/docs/overview.html +469 -0
  279. simstack/static/docs/py-modindex.html +1111 -0
  280. simstack/static/docs/recovery.html +371 -0
  281. simstack/static/docs/resource_config.html +590 -0
  282. simstack/static/docs/search.html +327 -0
  283. simstack/static/docs/searchindex.js +1 -0
  284. simstack/static/docs/simstack/modules/modules.html +489 -0
  285. simstack/static/docs/simstack/modules/simstack.core.html +1883 -0
  286. simstack/static/docs/simstack/modules/simstack.core.services.html +698 -0
  287. simstack/static/docs/simstack/modules/simstack.html +3060 -0
  288. simstack/static/docs/simstack/modules/simstack.methods.html +1150 -0
  289. simstack/static/docs/simstack/modules/simstack.models.html +13113 -0
  290. simstack/static/docs/simstack/modules/simstack.tables.html +635 -0
  291. simstack/static/docs/simstack/modules/simstack.util.html +3320 -0
  292. simstack/static/docs/submit.html +487 -0
  293. simstack/static/docs/tables.html +539 -0
  294. simstack/static/docs/testing_complex_nodes.html +458 -0
  295. simstack/static/docs/ui_documentation.html +788 -0
  296. simstack/static/docs/workflows.html +766 -0
  297. simstack/tables/__init__.py +0 -0
  298. simstack/tables/model_table.py +191 -0
  299. simstack/tables/node_children.py +219 -0
  300. simstack/tables/node_table.py +517 -0
  301. simstack/tables/table_builder.py +314 -0
  302. simstack/util/__init__.py +0 -0
  303. simstack/util/async_zip_utils.py +18 -0
  304. simstack/util/b64mixin.py +23 -0
  305. simstack/util/cleaned_json_schema.py +20 -0
  306. simstack/util/config_reader.py +179 -0
  307. simstack/util/custom_model_dump.py +75 -0
  308. simstack/util/database_information.py +160 -0
  309. simstack/util/db.py +1013 -0
  310. simstack/util/db_logger.py +117 -0
  311. simstack/util/default_from_dict.py +57 -0
  312. simstack/util/directory_iterator.py +216 -0
  313. simstack/util/docstring_parser.py +186 -0
  314. simstack/util/file_hashing.py +72 -0
  315. simstack/util/file_transfer_client.py +427 -0
  316. simstack/util/files.py +396 -0
  317. simstack/util/find_python_files.py +33 -0
  318. simstack/util/generate_ui_schema.py +80 -0
  319. simstack/util/generic_list_mixin.py +120 -0
  320. simstack/util/geometry_utils.py +118 -0
  321. simstack/util/get_module_path.py +20 -0
  322. simstack/util/git_pull.py +65 -0
  323. simstack/util/git_repository_status.py +95 -0
  324. simstack/util/import_module.py +63 -0
  325. simstack/util/importer.py +312 -0
  326. simstack/util/init_data_source.py +41 -0
  327. simstack/util/make_graph.py +107 -0
  328. simstack/util/make_table.py +837 -0
  329. simstack/util/mappings.py +41 -0
  330. simstack/util/minimal_route_finder.py +72 -0
  331. simstack/util/mock_db.py +144 -0
  332. simstack/util/module_path_checker.py +38 -0
  333. simstack/util/object_list_mixin.py +282 -0
  334. simstack/util/path_manager.py +223 -0
  335. simstack/util/project_root_finder.py +41 -0
  336. simstack/util/queue_watcher.py +262 -0
  337. simstack/util/resource_config.py +198 -0
  338. simstack/util/route_finder.py +74 -0
  339. simstack/util/runner_util.py +166 -0
  340. simstack/util/runner_utils.py +143 -0
  341. simstack/util/safe_code_executor.py +194 -0
  342. simstack/util/sample_config.py +98 -0
  343. simstack/util/sensitive_content_filter.py +48 -0
  344. simstack/util/setup_logging.py +49 -0
  345. simstack/util/submit_to_watchdog.py +106 -0
  346. simstack/util/toml_reader.py +142 -0
  347. simstack/util/transform_file_name.py +41 -0
  348. simstack/util/ui_tools.py +213 -0
  349. simstack-2.0.1.dist-info/METADATA +454 -0
  350. simstack-2.0.1.dist-info/RECORD +354 -0
  351. simstack-2.0.1.dist-info/WHEEL +5 -0
  352. simstack-2.0.1.dist-info/entry_points.txt +12 -0
  353. simstack-2.0.1.dist-info/licenses/LICENSE +21 -0
  354. simstack-2.0.1.dist-info/top_level.txt +1 -0
simstack/static/docs/testing_complex_nodes.html ADDED
@@ -0,0 +1,458 @@
1
+ <!doctype html>
2
+ <html class="no-js" lang="en" data-content_root="./">
3
+ <head><meta charset="utf-8">
4
+ <meta name="viewport" content="width=device-width,initial-scale=1">
5
+ <meta name="color-scheme" content="light dark"><meta name="viewport" content="width=device-width, initial-scale=1" />
6
+ <link rel="index" title="Index" href="genindex.html"><link rel="search" title="Search" href="search.html"><link rel="next" title="Database Migrations" href="database_migrations.html"><link rel="prev" title="Creating Docker Containers for SimStack Nodes" href="docker_nodes.html">
7
+
8
+ <!-- Generated with Sphinx 9.1.0 and Furo 2025.12.19 -->
9
+ <title>Testing Complex Nodes - SimStack II documentation</title>
10
+ <link rel="stylesheet" type="text/css" href="_static/pygments.css?v=d111a655" />
11
+ <link rel="stylesheet" type="text/css" href="_static/styles/furo.css?v=7bdb33bb" />
12
+ <link rel="stylesheet" type="text/css" href="_static/styles/furo-extensions.css?v=8dab3a3b" />
13
+ <link rel="stylesheet" type="text/css" href="_static/cobalt2-theme.css?v=2d651046" />
14
+ <link rel="stylesheet" type="text/css" href="_static/docs-build-badge.css?v=803faff8" />
15
+
16
+
17
+
18
+
19
+ <style>
20
+ body {
21
+ --color-code-background: #f2f2f2;
22
+ --color-code-foreground: #1e1e1e;
23
+
24
+ }
25
+ @media not print {
26
+ body[data-theme="dark"] {
27
+ --color-code-background: #202020;
28
+ --color-code-foreground: #d0d0d0;
29
+ --color-background-primary: #132738;
30
+ --color-background-secondary: #1e3a52;
31
+ --color-foreground-primary: #ffffff;
32
+ --color-foreground-secondary: #bac8d3;
33
+ --color-brand-primary: #22d5ff;
34
+ --color-brand-content: #0088ff;
35
+
36
+ }
37
+ @media (prefers-color-scheme: dark) {
38
+ body:not([data-theme="light"]) {
39
+ --color-code-background: #202020;
40
+ --color-code-foreground: #d0d0d0;
41
+ --color-background-primary: #132738;
42
+ --color-background-secondary: #1e3a52;
43
+ --color-foreground-primary: #ffffff;
44
+ --color-foreground-secondary: #bac8d3;
45
+ --color-brand-primary: #22d5ff;
46
+ --color-brand-content: #0088ff;
47
+
48
+ }
49
+ }
50
+ }
51
+ </style></head>
52
+ <body>
53
+
54
+ <script>
55
+ document.body.dataset.theme = localStorage.getItem("theme") || "auto";
56
+ </script>
57
+
58
+
59
+ <svg xmlns="http://www.w3.org/2000/svg" style="display: none;">
60
+ <symbol id="svg-toc" viewBox="0 0 24 24">
61
+ <title>Contents</title>
62
+ <svg stroke="currentColor" fill="currentColor" stroke-width="0" viewBox="0 0 1024 1024">
63
+ <path d="M408 442h480c4.4 0 8-3.6 8-8v-56c0-4.4-3.6-8-8-8H408c-4.4 0-8 3.6-8 8v56c0 4.4 3.6 8 8 8zm-8 204c0 4.4 3.6 8 8 8h480c4.4 0 8-3.6 8-8v-56c0-4.4-3.6-8-8-8H408c-4.4 0-8 3.6-8 8v56zm504-486H120c-4.4 0-8 3.6-8 8v56c0 4.4 3.6 8 8 8h784c4.4 0 8-3.6 8-8v-56c0-4.4-3.6-8-8-8zm0 632H120c-4.4 0-8 3.6-8 8v56c0 4.4 3.6 8 8 8h784c4.4 0 8-3.6 8-8v-56c0-4.4-3.6-8-8-8zM115.4 518.9L271.7 642c5.8 4.6 14.4.5 14.4-6.9V388.9c0-7.4-8.5-11.5-14.4-6.9L115.4 505.1a8.74 8.74 0 0 0 0 13.8z"/>
64
+ </svg>
65
+ </symbol>
66
+ <symbol id="svg-menu" viewBox="0 0 24 24">
67
+ <title>Menu</title>
68
+ <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
69
+ stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather-menu">
70
+ <line x1="3" y1="12" x2="21" y2="12"></line>
71
+ <line x1="3" y1="6" x2="21" y2="6"></line>
72
+ <line x1="3" y1="18" x2="21" y2="18"></line>
73
+ </svg>
74
+ </symbol>
75
+ <symbol id="svg-arrow-right" viewBox="0 0 24 24">
76
+ <title>Expand</title>
77
+ <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
78
+ stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather-chevron-right">
79
+ <polyline points="9 18 15 12 9 6"></polyline>
80
+ </svg>
81
+ </symbol>
82
+ <symbol id="svg-sun" viewBox="0 0 24 24">
83
+ <title>Light mode</title>
84
+ <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
85
+ stroke-width="1" stroke-linecap="round" stroke-linejoin="round" class="feather-sun">
86
+ <circle cx="12" cy="12" r="5"></circle>
87
+ <line x1="12" y1="1" x2="12" y2="3"></line>
88
+ <line x1="12" y1="21" x2="12" y2="23"></line>
89
+ <line x1="4.22" y1="4.22" x2="5.64" y2="5.64"></line>
90
+ <line x1="18.36" y1="18.36" x2="19.78" y2="19.78"></line>
91
+ <line x1="1" y1="12" x2="3" y2="12"></line>
92
+ <line x1="21" y1="12" x2="23" y2="12"></line>
93
+ <line x1="4.22" y1="19.78" x2="5.64" y2="18.36"></line>
94
+ <line x1="18.36" y1="5.64" x2="19.78" y2="4.22"></line>
95
+ </svg>
96
+ </symbol>
97
+ <symbol id="svg-moon" viewBox="0 0 24 24">
98
+ <title>Dark mode</title>
99
+ <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
100
+ stroke-width="1" stroke-linecap="round" stroke-linejoin="round" class="icon-tabler-moon">
101
+ <path stroke="none" d="M0 0h24v24H0z" fill="none" />
102
+ <path d="M12 3c.132 0 .263 0 .393 0a7.5 7.5 0 0 0 7.92 12.446a9 9 0 1 1 -8.313 -12.454z" />
103
+ </svg>
104
+ </symbol>
105
+ <symbol id="svg-sun-with-moon" viewBox="0 0 24 24">
106
+ <title>Auto light/dark, in light mode</title>
107
+ <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
108
+ stroke-width="1" stroke-linecap="round" stroke-linejoin="round"
109
+ class="icon-custom-derived-from-feather-sun-and-tabler-moon">
110
+ <path style="opacity: 50%" d="M 5.411 14.504 C 5.471 14.504 5.532 14.504 5.591 14.504 C 3.639 16.319 4.383 19.569 6.931 20.352 C 7.693 20.586 8.512 20.551 9.25 20.252 C 8.023 23.207 4.056 23.725 2.11 21.184 C 0.166 18.642 1.702 14.949 4.874 14.536 C 5.051 14.512 5.231 14.5 5.411 14.5 L 5.411 14.504 Z"/>
111
+ <line x1="14.5" y1="3.25" x2="14.5" y2="1.25"/>
112
+ <line x1="14.5" y1="15.85" x2="14.5" y2="17.85"/>
113
+ <line x1="10.044" y1="5.094" x2="8.63" y2="3.68"/>
114
+ <line x1="19" y1="14.05" x2="20.414" y2="15.464"/>
115
+ <line x1="8.2" y1="9.55" x2="6.2" y2="9.55"/>
116
+ <line x1="20.8" y1="9.55" x2="22.8" y2="9.55"/>
117
+ <line x1="10.044" y1="14.006" x2="8.63" y2="15.42"/>
118
+ <line x1="19" y1="5.05" x2="20.414" y2="3.636"/>
119
+ <circle cx="14.5" cy="9.55" r="3.6"/>
120
+ </svg>
121
+ </symbol>
122
+ <symbol id="svg-moon-with-sun" viewBox="0 0 24 24">
123
+ <title>Auto light/dark, in dark mode</title>
124
+ <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
125
+ stroke-width="1" stroke-linecap="round" stroke-linejoin="round"
126
+ class="icon-custom-derived-from-feather-sun-and-tabler-moon">
127
+ <path d="M 8.282 7.007 C 8.385 7.007 8.494 7.007 8.595 7.007 C 5.18 10.184 6.481 15.869 10.942 17.24 C 12.275 17.648 13.706 17.589 15 17.066 C 12.851 22.236 5.91 23.143 2.505 18.696 C -0.897 14.249 1.791 7.786 7.342 7.063 C 7.652 7.021 7.965 7 8.282 7 L 8.282 7.007 Z"/>
128
+ <line style="opacity: 50%" x1="18" y1="3.705" x2="18" y2="2.5"/>
129
+ <line style="opacity: 50%" x1="18" y1="11.295" x2="18" y2="12.5"/>
130
+ <line style="opacity: 50%" x1="15.316" y1="4.816" x2="14.464" y2="3.964"/>
131
+ <line style="opacity: 50%" x1="20.711" y1="10.212" x2="21.563" y2="11.063"/>
132
+ <line style="opacity: 50%" x1="14.205" y1="7.5" x2="13.001" y2="7.5"/>
133
+ <line style="opacity: 50%" x1="21.795" y1="7.5" x2="23" y2="7.5"/>
134
+ <line style="opacity: 50%" x1="15.316" y1="10.184" x2="14.464" y2="11.036"/>
135
+ <line style="opacity: 50%" x1="20.711" y1="4.789" x2="21.563" y2="3.937"/>
136
+ <circle style="opacity: 50%" cx="18" cy="7.5" r="2.169"/>
137
+ </svg>
138
+ </symbol>
139
+ <symbol id="svg-pencil" viewBox="0 0 24 24">
140
+ <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
141
+ stroke-width="1" stroke-linecap="round" stroke-linejoin="round" class="icon-tabler-pencil-code">
142
+ <path d="M4 20h4l10.5 -10.5a2.828 2.828 0 1 0 -4 -4l-10.5 10.5v4" />
143
+ <path d="M13.5 6.5l4 4" />
144
+ <path d="M20 21l2 -2l-2 -2" />
145
+ <path d="M17 17l-2 2l2 2" />
146
+ </svg>
147
+ </symbol>
148
+ <symbol id="svg-eye" viewBox="0 0 24 24">
149
+ <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
150
+ stroke-width="1" stroke-linecap="round" stroke-linejoin="round" class="icon-tabler-eye-code">
151
+ <path stroke="none" d="M0 0h24v24H0z" fill="none" />
152
+ <path d="M10 12a2 2 0 1 0 4 0a2 2 0 0 0 -4 0" />
153
+ <path
154
+ d="M11.11 17.958c-3.209 -.307 -5.91 -2.293 -8.11 -5.958c2.4 -4 5.4 -6 9 -6c3.6 0 6.6 2 9 6c-.21 .352 -.427 .688 -.647 1.008" />
155
+ <path d="M20 21l2 -2l-2 -2" />
156
+ <path d="M17 17l-2 2l2 2" />
157
+ </svg>
158
+ </symbol>
159
+ </svg>
160
+
161
+ <input type="checkbox" class="sidebar-toggle" name="__navigation" id="__navigation" aria-label="Toggle site navigation sidebar">
162
+ <input type="checkbox" class="sidebar-toggle" name="__toc" id="__toc" aria-label="Toggle table of contents sidebar">
163
+ <label class="overlay sidebar-overlay" for="__navigation"></label>
164
+ <label class="overlay toc-overlay" for="__toc"></label>
165
+
166
+ <a class="skip-to-content muted-link" href="#furo-main-content">Skip to content</a>
167
+
168
+
169
+
170
+ <div class="page">
171
+ <header class="mobile-header">
172
+ <div class="header-left">
173
+ <label class="nav-overlay-icon" for="__navigation">
174
+ <span class="icon"><svg><use href="#svg-menu"></use></svg></span>
175
+ </label>
176
+ </div>
177
+ <div class="header-center">
178
+ <a href="index.html"><div class="brand">SimStack II documentation</div></a>
179
+ </div>
180
+ <div class="header-right">
181
+ <div class="theme-toggle-container theme-toggle-header">
182
+ <button class="theme-toggle" aria-label="Toggle Light / Dark / Auto color theme">
183
+ <svg class="theme-icon-when-auto-light"><use href="#svg-sun-with-moon"></use></svg>
184
+ <svg class="theme-icon-when-auto-dark"><use href="#svg-moon-with-sun"></use></svg>
185
+ <svg class="theme-icon-when-dark"><use href="#svg-moon"></use></svg>
186
+ <svg class="theme-icon-when-light"><use href="#svg-sun"></use></svg>
187
+ </button>
188
+ </div>
189
+ <label class="toc-overlay-icon toc-header-icon" for="__toc">
190
+ <span class="icon"><svg><use href="#svg-toc"></use></svg></span>
191
+ </label>
192
+ </div>
193
+ </header>
194
+ <aside class="sidebar-drawer">
195
+ <div class="sidebar-container">
196
+
197
+ <div class="sidebar-sticky"><a class="sidebar-brand" href="index.html">
198
+
199
+
200
+ </a><form class="sidebar-search-container" method="get" action="search.html" role="search">
201
+ <input class="sidebar-search" placeholder="Search" name="q" aria-label="Search">
202
+ <input type="hidden" name="check_keywords" value="yes">
203
+ <input type="hidden" name="area" value="default">
204
+ </form>
205
+ <div id="searchbox"></div><div class="sidebar-scroll"><div class="sidebar-tree">
206
+ <p class="caption" role="heading"><span class="caption-text">Contents:</span></p>
207
+ <ul class="current">
208
+ <li class="toctree-l1"><a class="reference internal" href="overview.html">Overview</a></li>
209
+ <li class="toctree-l1"><a class="reference internal" href="workflows.html">Workflows</a></li>
210
+ <li class="toctree-l1"><a class="reference internal" href="ui_documentation.html">Graphical User Interface</a></li>
211
+ <li class="toctree-l1 current has-children"><a class="reference internal" href="developer_guide.html">Developer Guide</a><input aria-label="Toggle navigation of Developer Guide" checked="" class="toctree-checkbox" id="toctree-checkbox-1" name="toctree-checkbox-1" role="switch" type="checkbox"/><label for="toctree-checkbox-1"><span class="icon"><svg><use href="#svg-arrow-right"></use></svg></span></label><ul class="current">
212
+ <li class="toctree-l2"><a class="reference internal" href="data.html">Persisting and Visualizing Data</a></li>
213
+ <li class="toctree-l2"><a class="reference internal" href="datasets.html">Datasets</a></li>
214
+ <li class="toctree-l2"><a class="reference internal" href="lists.html">List Mixins</a></li>
215
+ <li class="toctree-l2"><a class="reference internal" href="array_lists.html">Array Lists</a></li>
216
+ <li class="toctree-l2"><a class="reference internal" href="tables.html">Tables</a></li>
217
+ <li class="toctree-l2"><a class="reference internal" href="files.html">File Management</a></li>
218
+ <li class="toctree-l2"><a class="reference internal" href="resource_config.html">Program Execution</a></li>
219
+ <li class="toctree-l2"><a class="reference internal" href="mass_runner.html">MassRunner</a></li>
220
+ <li class="toctree-l2"><a class="reference internal" href="fire_and_forget_runner.html">FireAndForgetRunner</a></li>
221
+ <li class="toctree-l2"><a class="reference internal" href="docker_nodes.html">Creating Docker Containers for SimStack Nodes</a></li>
222
+ <li class="toctree-l2 current current-page"><a class="current reference internal" href="#">Testing Complex Nodes</a></li>
223
+ <li class="toctree-l2"><a class="reference internal" href="database_migrations.html">Database Migrations</a></li>
224
+ <li class="toctree-l2"><a class="reference internal" href="ci_pipeline.html">CI Pipeline and Documentation Artifacts</a></li>
225
+ <li class="toctree-l2"><a class="reference internal" href="misc.html">Miscellaneous</a></li>
226
+ </ul>
227
+ </li>
228
+ <li class="toctree-l1"><a class="reference internal" href="installation.html">Simstack II Setup</a></li>
229
+ <li class="toctree-l1 has-children"><a class="reference internal" href="simstack/modules/modules.html">simstack</a><input aria-label="Toggle navigation of simstack" class="toctree-checkbox" id="toctree-checkbox-2" name="toctree-checkbox-2" role="switch" type="checkbox"/><label for="toctree-checkbox-2"><span class="icon"><svg><use href="#svg-arrow-right"></use></svg></span></label><ul>
230
+ <li class="toctree-l2 has-children"><a class="reference internal" href="simstack/modules/simstack.html">simstack package</a><input aria-label="Toggle navigation of simstack package" class="toctree-checkbox" id="toctree-checkbox-3" name="toctree-checkbox-3" role="switch" type="checkbox"/><label for="toctree-checkbox-3"><span class="icon"><svg><use href="#svg-arrow-right"></use></svg></span></label><ul>
231
+ <li class="toctree-l3 has-children"><a class="reference internal" href="simstack/modules/simstack.core.html">simstack.core package</a><input aria-label="Toggle navigation of simstack.core package" class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" role="switch" type="checkbox"/><label for="toctree-checkbox-4"><span class="icon"><svg><use href="#svg-arrow-right"></use></svg></span></label><ul>
232
+ <li class="toctree-l4"><a class="reference internal" href="simstack/modules/simstack.core.services.html">simstack.core.services package</a></li>
233
+ </ul>
234
+ </li>
235
+ <li class="toctree-l3"><a class="reference internal" href="simstack/modules/simstack.methods.html">simstack.methods package</a></li>
236
+ <li class="toctree-l3"><a class="reference internal" href="simstack/modules/simstack.models.html">simstack.models package</a></li>
237
+ <li class="toctree-l3"><a class="reference internal" href="simstack/modules/simstack.tables.html">simstack.tables package</a></li>
238
+ <li class="toctree-l3"><a class="reference internal" href="simstack/modules/simstack.util.html">simstack.util package</a></li>
239
+ </ul>
240
+ </li>
241
+ </ul>
242
+ </li>
243
+ </ul>
244
+
245
+ </div>
246
+ </div>
247
+
248
+ </div>
249
+
250
+ </div>
251
+ </aside>
252
+ <div class="main">
253
+ <div class="content">
254
+ <div class="article-container">
255
+ <a href="#" class="back-to-top muted-link">
256
+ <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24">
257
+ <path d="M13 20h-2V8l-5.5 5.5-1.42-1.42L12 4.16l7.92 7.92-1.42 1.42L13 8v12z"></path>
258
+ </svg>
259
+ <span>Back to top</span>
260
+ </a>
261
+ <div class="content-icon-container">
262
+ <div class="view-this-page">
263
+ <a class="muted-link" href="_sources/testing_complex_nodes.rst.txt" title="View this page">
264
+ <svg><use href="#svg-eye"></use></svg>
265
+ <span class="visually-hidden">View this page</span>
266
+ </a>
267
+ </div>
268
+ <div class="theme-toggle-container theme-toggle-content">
269
+ <button class="theme-toggle" aria-label="Toggle Light / Dark / Auto color theme">
270
+ <svg class="theme-icon-when-auto-light"><use href="#svg-sun-with-moon"></use></svg>
271
+ <svg class="theme-icon-when-auto-dark"><use href="#svg-moon-with-sun"></use></svg>
272
+ <svg class="theme-icon-when-dark"><use href="#svg-moon"></use></svg>
273
+ <svg class="theme-icon-when-light"><use href="#svg-sun"></use></svg>
274
+ </button>
275
+ </div>
276
+ <label class="toc-overlay-icon toc-content-icon" for="__toc">
277
+ <span class="icon"><svg><use href="#svg-toc"></use></svg></span>
278
+ </label>
279
+ </div>
280
+ <article role="main" id="furo-main-content">
281
+ <section id="testing-complex-nodes">
282
+ <h1>Testing Complex Nodes<a class="headerlink" href="#testing-complex-nodes" title="Link to this heading">¶</a></h1>
283
+ <p>This document explains the testing strategy for complex nodes in SimStack, focusing on the test generation and re-execution mechanism.</p>
284
+ <section id="overview">
285
+ <h2>Overview<a class="headerlink" href="#overview" title="Link to this heading">¶</a></h2>
286
+ <p>In SimStack, some nodes perform complex tasks that might be difficult to reproduce in a standard unit test environment due to external dependencies, large data volumes, or non-deterministic behavior. To address this, we use a strategy of capturing the state of a successful node execution and using it to generate a repeatable test case.</p>
287
+ <p>The two main components of this strategy are:
288
+ 1. <cite>simstack/src/simstack/methods/generate_test.py</cite>: A tool to extract node data from the database and work directory.
289
+ 2. <cite>examples/testing/testing_test_generation.py</cite>: An example script demonstrating how to use generated tests and patch complex executions.</p>
290
+ </section>
291
+ <section id="test-generation-generate-test-py">
292
+ <h2>Test Generation (<cite>generate_test.py</cite>)<a class="headerlink" href="#test-generation-generate-test-py" title="Link to this heading">¶</a></h2>
293
+ <p>The <cite>generate_test</cite> method (and its CLI wrapper) allows developers to create a test case from an existing <cite>NodeRegistry</cite> entry.</p>
294
+ <p>Workflow:
295
+ 1. <strong>Identify Node</strong>: The user provides a <cite>node_id</cite> (ObjectId of a <cite>NodeRegistry</cite> entry).
296
+ 2. <strong>Setup Target</strong>: A target directory is created, organized by node name and argument hash: <cite>target_base / node_name / arg_hash</cite>.
297
+ 3. <strong>Capture Files</strong>: All files from the node’s work directory (<cite>workdir / node_name / node_id</cite>) are copied to the target directory.
298
+ 4. <strong>Serialize Data</strong>:</p>
299
+ <blockquote>
300
+ <div><ul class="simple">
301
+ <li><p>Input models are loaded from the database based on <cite>input_references</cite> and serialized to <cite>inputs.json</cite>.</p></li>
302
+ <li><p>Output models (results) are loaded based on <cite>results_references</cite> and serialized to <cite>outputs.json</cite>.</p></li>
303
+ <li><p>This is for convenience and not really needed for the test</p></li>
304
+ </ul>
305
+ </div></blockquote>
306
+ <p>This results in a self-contained directory containing everything needed to verify the node’s behavior for a specific set of inputs.</p>
307
+ </section>
308
+ <section id="testing-with-patches-testing-test-generation-py">
309
+ <h2>Testing with Patches (<cite>testing_test_generation.py</cite>)<a class="headerlink" href="#testing-with-patches-testing-test-generation-py" title="Link to this heading">¶</a></h2>
310
+ <p>Once a test case is generated, it can be used to verify that changes to the node’s logic don’t break existing functionality, or to debug complex failures.</p>
311
+ <p>The example script <cite>testing_test_generation.py</cite> demonstrates a pattern for testing nodes that call external functions or have complex side effects.</p>
312
+ <p>### The Problem: Complex Side Effects
313
+ Some nodes call functions that perform operations (like submitting a job to a cluster) which cannot be easily duplicated in testing.
314
+ The idea is to store all of the resulting inout and output files and also serialized versions of all costly intermediate results.
315
+ During a test run, we don’t want to actually submit a new job; instead, we want to simulate the previous execution and
316
+ return the <em>original</em> <cite>task_id</cite> so that SimStack can find the associated files and results.</p>
317
+ <p>### The Solution: Patching and Re-execution
318
+ The strategy involves patching the “inner” execution function during the test run.</p>
319
+ <ol class="arabic">
320
+ <li><p><strong>Setup</strong>: The node is first executed normally (or a previous execution is identified). All of the complex and costly computations are performed
321
+ in a function like <cite>complex_task_execution</cite> which <strong>must</strong> have the signature <a href="#id1"><span class="problematic" id="id2">*</span></a>args, <a href="#id3"><span class="problematic" id="id4">**</span></a>kwargs.</p>
322
+ <blockquote>
323
+ <div><ul class="simple">
324
+ <li><p>The <a href="#id5"><span class="problematic" id="id6">*</span></a>args are used during normal execution to generate the costly data and serialize it.</p></li>
325
+ <li><p>The function returns data needed to locally construct the output models. Generating the input files or arsing the output files of a job, which is
326
+ part of a test, should be outside <cite>complex_task_execution</cite></p></li>
327
+ <li><p>The goal is: Do all the stuff that can be locally and that needs to be testes outside of <a href="#id7"><span class="problematic" id="id8">`</span></a>complex_task_execution</p></li>
328
+ </ul>
329
+ </div></blockquote>
330
+ </li>
331
+ <li><p><strong>Generation</strong>: <cite>generate_test</cite> is called to create the test data.</p></li>
332
+ <li><dl class="simple">
333
+ <dt><strong>Patching</strong>: Using <cite>unittest.mock.patch</cite>, the function responsible for the side effect (e.g., <cite>complex_task_execution</cite>) is replaced with</dt><dd><p>a <cite>patched_complex_task_execution</cite>. This function also gets the <a href="#id9"><span class="problematic" id="id10">*</span></a>args and <a href="#id11"><span class="problematic" id="id12">**</span></a>kwargs. It needs the latter to automatically find
334
+ the test directory with respect to a test-directory root.</p>
335
+ </dd>
336
+ </dl>
337
+ </li>
338
+ <li><p><strong>Mock Logic</strong>: The patched function:
339
+ - Identifies the correct test directory using <cite>arg_hash</cite> and node name (provided in <cite>kwargs</cite>).
340
+ - The file can compare the input files generated in the test run with the input files in the test directory
341
+ - Restores the necessary files (e.g., <cite>test.txt</cite>) from the test directory to the current execution environment.
342
+ - Deserialize all important information to locally build the outputs of the nodes. In the test this is only the original <cite>task_id</cite>.</p></li>
343
+ <li><p><strong>Re-execution</strong>: The node is called again with <cite>force_rerun=True</cite>.</p></li>
344
+ <li><p><strong>Verification</strong>: The results of the re-execution are compared against the original results to ensure consistency.</p></li>
345
+ </ol>
346
+ <section id="example-patch-implementation">
347
+ <h3>Example Patch Implementation<a class="headerlink" href="#example-patch-implementation" title="Link to this heading">¶</a></h3>
348
+ <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="k">def</span><span class="w"> </span><span class="nf">patched_complex_task_execution</span><span class="p">(</span><span class="n">arg_patched</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs_patched</span><span class="p">):</span>
349
+ <span class="c1"># Locate the test directory</span>
350
+ <span class="n">test_dir</span> <span class="o">=</span> <span class="n">local_target_base</span> <span class="o">/</span> <span class="n">node_name</span> <span class="o">/</span> <span class="n">kwargs_patched</span><span class="p">[</span><span class="s2">&quot;arg_hash&quot;</span><span class="p">]</span>
351
+
352
+ <span class="c1"># Restore state</span>
353
+ <span class="n">shutil</span><span class="o">.</span><span class="n">copy2</span><span class="p">(</span><span class="n">test_dir</span> <span class="o">/</span> <span class="s2">&quot;test.txt&quot;</span><span class="p">,</span> <span class="n">Path</span><span class="o">.</span><span class="n">cwd</span><span class="p">()</span> <span class="o">/</span> <span class="s2">&quot;test.txt&quot;</span><span class="p">)</span>
354
+
355
+ <span class="c1"># Retrieve original ID</span>
356
+ <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="s2">&quot;test.txt&quot;</span><span class="p">,</span> <span class="s2">&quot;r&quot;</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
357
+ <span class="n">original_task_id</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">readlines</span><span class="p">()[</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
358
+
359
+ <span class="k">return</span> <span class="n">original_task_id</span>
360
+
361
+ <span class="k">with</span> <span class="n">patch</span><span class="p">(</span><span class="s2">&quot;__main__.complex_task_execution&quot;</span><span class="p">,</span> <span class="n">side_effect</span><span class="o">=</span><span class="n">patched_complex_task_execution</span><span class="p">):</span>
362
+ <span class="n">test_result</span> <span class="o">=</span> <span class="n">complex_task_for_testing</span><span class="p">(</span><span class="n">arg</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="n">Parameters</span><span class="p">(</span><span class="n">force_rerun</span><span class="o">=</span><span class="kc">True</span><span class="p">))</span>
363
+ <span class="c1"># Assertions...</span>
364
+ </pre></div>
365
+ </div>
366
+ </section>
367
+ </section>
368
+ <section id="benefits">
369
+ <h2>Benefits<a class="headerlink" href="#benefits" title="Link to this heading">¶</a></h2>
370
+ <ul class="simple">
371
+ <li><p><strong>Reproducibility</strong>: Easily reproduce complex node executions in a local environment.</p></li>
372
+ <li><p><strong>Isolation</strong>: Test node logic without triggering expensive or external side effects.</p></li>
373
+ <li><p><strong>Verification</strong>: Ensure that data processing remains consistent over time.</p></li>
374
+ </ul>
375
+ </section>
376
+ </section>
377
+
378
+ </article>
379
+ </div>
380
+ <footer>
381
+
382
+ <div class="related-pages">
383
+ <a class="next-page" href="database_migrations.html">
384
+ <div class="page-info">
385
+ <div class="context">
386
+ <span>Next</span>
387
+ </div>
388
+ <div class="title">Database Migrations</div>
389
+ </div>
390
+ <svg class="furo-related-icon"><use href="#svg-arrow-right"></use></svg>
391
+ </a>
392
+ <a class="prev-page" href="docker_nodes.html">
393
+ <svg class="furo-related-icon"><use href="#svg-arrow-right"></use></svg>
394
+ <div class="page-info">
395
+ <div class="context">
396
+ <span>Previous</span>
397
+ </div>
398
+
399
+ <div class="title">Creating Docker Containers for SimStack Nodes</div>
400
+
401
+ </div>
402
+ </a>
403
+ </div>
404
+ <div class="bottom-of-page">
405
+ <div class="left-details">
406
+ <div class="copyright">
407
+ Copyright &#169; 2026, Wolfgang Wenzel
408
+ </div>
409
+ Made with <a href="https://www.sphinx-doc.org/">Sphinx</a> and <a class="muted-link" href="https://pradyunsg.me">@pradyunsg</a>'s
410
+
411
+ <a href="https://github.com/pradyunsg/furo">Furo</a>
412
+
413
+ </div>
414
+ <div class="right-details">
415
+
416
+ </div>
417
+ </div>
418
+
419
+ </footer>
420
+ </div>
421
+ <aside class="toc-drawer">
422
+
423
+
424
+ <div class="toc-sticky toc-scroll">
425
+ <div class="toc-title-container">
426
+ <span class="toc-title">
427
+ On this page
428
+ </span>
429
+ </div>
430
+ <div class="toc-tree-container">
431
+ <div class="toc-tree">
432
+ <ul>
433
+ <li><a class="reference internal" href="#">Testing Complex Nodes</a><ul>
434
+ <li><a class="reference internal" href="#overview">Overview</a></li>
435
+ <li><a class="reference internal" href="#test-generation-generate-test-py">Test Generation (<cite>generate_test.py</cite>)</a></li>
436
+ <li><a class="reference internal" href="#testing-with-patches-testing-test-generation-py">Testing with Patches (<cite>testing_test_generation.py</cite>)</a><ul>
437
+ <li><a class="reference internal" href="#example-patch-implementation">Example Patch Implementation</a></li>
438
+ </ul>
439
+ </li>
440
+ <li><a class="reference internal" href="#benefits">Benefits</a></li>
441
+ </ul>
442
+ </li>
443
+ </ul>
444
+
445
+ </div>
446
+ </div>
447
+ </div>
448
+
449
+
450
+ </aside>
451
+ </div>
452
+ </div><script src="_static/documentation_options.js?v=5929fcd5"></script>
453
+ <script src="_static/doctools.js?v=fd6eb6e6"></script>
454
+ <script src="_static/sphinx_highlight.js?v=6ffebe34"></script>
455
+ <script src="_static/scripts/furo.js?v=46bd48cc"></script>
456
+ <script data-docs-built-at="2026-07-17 10:50:10 UTC" src="_static/docs-build-badge.js?v=57cb9fc6"></script>
457
+ </body>
458
+ </html>