simstack 2.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (354) hide show
  1. simstack/__init__.py +0 -0
  2. simstack/_version.py +24 -0
  3. simstack/core/__init__.py +0 -0
  4. simstack/core/artifacts.py +406 -0
  5. simstack/core/asnyc_helper.py +14 -0
  6. simstack/core/config_file.py +60 -0
  7. simstack/core/context.py +283 -0
  8. simstack/core/definitions.py +22 -0
  9. simstack/core/engine.py +115 -0
  10. simstack/core/find_simstack_modules.py +41 -0
  11. simstack/core/hash.py +179 -0
  12. simstack/core/node.py +1137 -0
  13. simstack/core/node_claim.py +31 -0
  14. simstack/core/node_runner.py +362 -0
  15. simstack/core/process_results.py +102 -0
  16. simstack/core/recompute_artifacts.py +117 -0
  17. simstack/core/reset_db.py +12 -0
  18. simstack/core/resource_assignment.py +222 -0
  19. simstack/core/resources.py +90 -0
  20. simstack/core/route_table.py +24 -0
  21. simstack/core/run_docker.py +109 -0
  22. simstack/core/run_node.py +71 -0
  23. simstack/core/runner.py +169 -0
  24. simstack/core/services/__init__.py +11 -0
  25. simstack/core/services/base_service.py +191 -0
  26. simstack/core/services/file_transfer_service.py +117 -0
  27. simstack/core/services/git_restart_service.py +12 -0
  28. simstack/core/services/git_uv_update_service.py +105 -0
  29. simstack/core/services/node_execution_service.py +182 -0
  30. simstack/core/services/resource_branch_monitor_service.py +71 -0
  31. simstack/core/services/runner_cleanup_service.py +149 -0
  32. simstack/core/services/runner_manager.py +163 -0
  33. simstack/core/services/runner_status_service.py +10 -0
  34. simstack/core/services/slurm_status_service.py +67 -0
  35. simstack/core/services/stop_check_service.py +27 -0
  36. simstack/core/services/timeout_restart_service.py +26 -0
  37. simstack/core/simstack_result.py +39 -0
  38. simstack/core/submit_node.py +217 -0
  39. simstack/core/task_id.py +22 -0
  40. simstack/methods/__init__.py +0 -0
  41. simstack/methods/archive_files.py +292 -0
  42. simstack/methods/fire_and_forget_runner.py +113 -0
  43. simstack/methods/generate_test.py +125 -0
  44. simstack/methods/get_file.py +26 -0
  45. simstack/methods/mass_runner.py +154 -0
  46. simstack/methods/multiple_executor.py +307 -0
  47. simstack/methods/switch_git.py +16 -0
  48. simstack/methods/upload_helpers.py +81 -0
  49. simstack/models/__init__.py +74 -0
  50. simstack/models/array_list.py +21 -0
  51. simstack/models/array_storage.py +83 -0
  52. simstack/models/artifact_models.py +127 -0
  53. simstack/models/base_lists.py +51 -0
  54. simstack/models/base_types.py +209 -0
  55. simstack/models/charts_artifact.py +690 -0
  56. simstack/models/dataset.py +451 -0
  57. simstack/models/dataset_metadata.py +390 -0
  58. simstack/models/datasettuple.py +587 -0
  59. simstack/models/file_instance.py +150 -0
  60. simstack/models/file_list.py +45 -0
  61. simstack/models/files.py +424 -0
  62. simstack/models/fire_and_forget_result.py +19 -0
  63. simstack/models/images2d.py +113 -0
  64. simstack/models/log_entry_model.py +37 -0
  65. simstack/models/models.py +52 -0
  66. simstack/models/named_data_reference.py +36 -0
  67. simstack/models/node_registry.py +96 -0
  68. simstack/models/pandas_model.py +239 -0
  69. simstack/models/parameters.py +402 -0
  70. simstack/models/pickle_models.py +300 -0
  71. simstack/models/project.py +13 -0
  72. simstack/models/resource_assignment.py +170 -0
  73. simstack/models/resource_definition.py +125 -0
  74. simstack/models/runner_model.py +39 -0
  75. simstack/models/simple_table.py +79 -0
  76. simstack/models/simstack_model.py +97 -0
  77. simstack/models/slurm_info.py +17 -0
  78. simstack/models/table_artifact.py +87 -0
  79. simstack/models/tag.py +12 -0
  80. simstack/models/unused_test_file.py +5 -0
  81. simstack/simstack_marker.com +1 -0
  82. simstack/static/docs/.buildinfo +4 -0
  83. simstack/static/docs/_images/GUI Concept.webp +0 -0
  84. simstack/static/docs/_modules/index.html +434 -0
  85. simstack/static/docs/_modules/simstack/core/artifacts.html +743 -0
  86. simstack/static/docs/_modules/simstack/core/asnyc_helper.html +330 -0
  87. simstack/static/docs/_modules/simstack/core/config_file.html +379 -0
  88. simstack/static/docs/_modules/simstack/core/definitions.html +341 -0
  89. simstack/static/docs/_modules/simstack/core/engine.html +437 -0
  90. simstack/static/docs/_modules/simstack/core/find_simstack_modules.html +360 -0
  91. simstack/static/docs/_modules/simstack/core/hash.html +528 -0
  92. simstack/static/docs/_modules/simstack/core/node.html +1392 -0
  93. simstack/static/docs/_modules/simstack/core/node_claim.html +347 -0
  94. simstack/static/docs/_modules/simstack/core/node_runner.html +711 -0
  95. simstack/static/docs/_modules/simstack/core/process_results.html +418 -0
  96. simstack/static/docs/_modules/simstack/core/recompute_artifacts.html +433 -0
  97. simstack/static/docs/_modules/simstack/core/reset_db.html +328 -0
  98. simstack/static/docs/_modules/simstack/core/resource_assignment.html +553 -0
  99. simstack/static/docs/_modules/simstack/core/resources.html +424 -0
  100. simstack/static/docs/_modules/simstack/core/route_table.html +349 -0
  101. simstack/static/docs/_modules/simstack/core/run_docker.html +425 -0
  102. simstack/static/docs/_modules/simstack/core/run_node.html +390 -0
  103. simstack/static/docs/_modules/simstack/core/runner.html +491 -0
  104. simstack/static/docs/_modules/simstack/core/services/base_service.html +522 -0
  105. simstack/static/docs/_modules/simstack/core/services/file_transfer_service.html +436 -0
  106. simstack/static/docs/_modules/simstack/core/services/git_restart_service.html +331 -0
  107. simstack/static/docs/_modules/simstack/core/services/git_uv_update_service.html +427 -0
  108. simstack/static/docs/_modules/simstack/core/services/node_execution_service.html +507 -0
  109. simstack/static/docs/_modules/simstack/core/services/resource_branch_monitor_service.html +390 -0
  110. simstack/static/docs/_modules/simstack/core/services/runner_cleanup_service.html +468 -0
  111. simstack/static/docs/_modules/simstack/core/services/runner_manager.html +485 -0
  112. simstack/static/docs/_modules/simstack/core/services/runner_status_service.html +329 -0
  113. simstack/static/docs/_modules/simstack/core/services/slurm_status_service.html +386 -0
  114. simstack/static/docs/_modules/simstack/core/services/stop_check_service.html +346 -0
  115. simstack/static/docs/_modules/simstack/core/services/timeout_restart_service.html +345 -0
  116. simstack/static/docs/_modules/simstack/core/simstack_result.html +355 -0
  117. simstack/static/docs/_modules/simstack/core/submit_node.html +536 -0
  118. simstack/static/docs/_modules/simstack/core/task_id.html +344 -0
  119. simstack/static/docs/_modules/simstack/methods/archive_files.html +598 -0
  120. simstack/static/docs/_modules/simstack/methods/fire_and_forget_runner.html +432 -0
  121. simstack/static/docs/_modules/simstack/methods/generate_test.html +443 -0
  122. simstack/static/docs/_modules/simstack/methods/get_file.html +342 -0
  123. simstack/static/docs/_modules/simstack/methods/mass_runner.html +476 -0
  124. simstack/static/docs/_modules/simstack/methods/multiple_executor.html +674 -0
  125. simstack/static/docs/_modules/simstack/methods/switch_git.html +332 -0
  126. simstack/static/docs/_modules/simstack/methods/upload_helpers.html +403 -0
  127. simstack/static/docs/_modules/simstack/models/array_list.html +337 -0
  128. simstack/static/docs/_modules/simstack/models/array_storage.html +417 -0
  129. simstack/static/docs/_modules/simstack/models/base_lists.html +374 -0
  130. simstack/static/docs/_modules/simstack/models/base_types.html +588 -0
  131. simstack/static/docs/_modules/simstack/models/charts_artifact.html +1035 -0
  132. simstack/static/docs/_modules/simstack/models/dataset.html +866 -0
  133. simstack/static/docs/_modules/simstack/models/dataset_metadata.html +711 -0
  134. simstack/static/docs/_modules/simstack/models/datasettuple.html +1005 -0
  135. simstack/static/docs/_modules/simstack/models/file_instance.html +472 -0
  136. simstack/static/docs/_modules/simstack/models/file_list.html +367 -0
  137. simstack/static/docs/_modules/simstack/models/files.html +773 -0
  138. simstack/static/docs/_modules/simstack/models/fire_and_forget_result.html +335 -0
  139. simstack/static/docs/_modules/simstack/models/images2d.html +447 -0
  140. simstack/static/docs/_modules/simstack/models/log_entry_model.html +356 -0
  141. simstack/static/docs/_modules/simstack/models/models.html +374 -0
  142. simstack/static/docs/_modules/simstack/models/named_data_reference.html +355 -0
  143. simstack/static/docs/_modules/simstack/models/node_registry.html +415 -0
  144. simstack/static/docs/_modules/simstack/models/pandas_model.html +570 -0
  145. simstack/static/docs/_modules/simstack/models/parameters.html +739 -0
  146. simstack/static/docs/_modules/simstack/models/pickle_models.html +631 -0
  147. simstack/static/docs/_modules/simstack/models/project.html +329 -0
  148. simstack/static/docs/_modules/simstack/models/resource_assignment.html +501 -0
  149. simstack/static/docs/_modules/simstack/models/resource_definition.html +477 -0
  150. simstack/static/docs/_modules/simstack/models/runner_model.html +361 -0
  151. simstack/static/docs/_modules/simstack/models/simple_table.html +413 -0
  152. simstack/static/docs/_modules/simstack/models/simstack_model.html +416 -0
  153. simstack/static/docs/_modules/simstack/models/slurm_info.html +333 -0
  154. simstack/static/docs/_modules/simstack/models/table_artifact.html +406 -0
  155. simstack/static/docs/_modules/simstack/models/tag.html +328 -0
  156. simstack/static/docs/_modules/simstack/tables/model_table.html +519 -0
  157. simstack/static/docs/_modules/simstack/tables/node_children.html +538 -0
  158. simstack/static/docs/_modules/simstack/tables/node_table.html +854 -0
  159. simstack/static/docs/_modules/simstack/tables/table_builder.html +642 -0
  160. simstack/static/docs/_modules/simstack/util/async_zip_utils.html +337 -0
  161. simstack/static/docs/_modules/simstack/util/b64mixin.html +339 -0
  162. simstack/static/docs/_modules/simstack/util/cleaned_json_schema.html +336 -0
  163. simstack/static/docs/_modules/simstack/util/config_reader.html +498 -0
  164. simstack/static/docs/_modules/simstack/util/custom_model_dump.html +391 -0
  165. simstack/static/docs/_modules/simstack/util/database_information.html +488 -0
  166. simstack/static/docs/_modules/simstack/util/db.html +1392 -0
  167. simstack/static/docs/_modules/simstack/util/db_logger.html +448 -0
  168. simstack/static/docs/_modules/simstack/util/default_from_dict.html +376 -0
  169. simstack/static/docs/_modules/simstack/util/directory_iterator.html +553 -0
  170. simstack/static/docs/_modules/simstack/util/docstring_parser.html +526 -0
  171. simstack/static/docs/_modules/simstack/util/file_hashing.html +394 -0
  172. simstack/static/docs/_modules/simstack/util/file_transfer_client.html +791 -0
  173. simstack/static/docs/_modules/simstack/util/files.html +733 -0
  174. simstack/static/docs/_modules/simstack/util/find_python_files.html +349 -0
  175. simstack/static/docs/_modules/simstack/util/generate_ui_schema.html +396 -0
  176. simstack/static/docs/_modules/simstack/util/generic_list_mixin.html +487 -0
  177. simstack/static/docs/_modules/simstack/util/get_module_path.html +336 -0
  178. simstack/static/docs/_modules/simstack/util/git_pull.html +381 -0
  179. simstack/static/docs/_modules/simstack/util/git_repository_status.html +411 -0
  180. simstack/static/docs/_modules/simstack/util/import_module.html +377 -0
  181. simstack/static/docs/_modules/simstack/util/importer.html +637 -0
  182. simstack/static/docs/_modules/simstack/util/init_data_source.html +357 -0
  183. simstack/static/docs/_modules/simstack/util/make_graph.html +423 -0
  184. simstack/static/docs/_modules/simstack/util/make_table.html +1171 -0
  185. simstack/static/docs/_modules/simstack/util/mappings.html +378 -0
  186. simstack/static/docs/_modules/simstack/util/minimal_route_finder.html +388 -0
  187. simstack/static/docs/_modules/simstack/util/mock_db.html +472 -0
  188. simstack/static/docs/_modules/simstack/util/module_path_checker.html +354 -0
  189. simstack/static/docs/_modules/simstack/util/object_list_mixin.html +649 -0
  190. simstack/static/docs/_modules/simstack/util/path_manager.html +566 -0
  191. simstack/static/docs/_modules/simstack/util/project_root_finder.html +357 -0
  192. simstack/static/docs/_modules/simstack/util/queue_watcher.html +593 -0
  193. simstack/static/docs/_modules/simstack/util/resource_config.html +532 -0
  194. simstack/static/docs/_modules/simstack/util/route_finder.html +390 -0
  195. simstack/static/docs/_modules/simstack/util/runner_util.html +487 -0
  196. simstack/static/docs/_modules/simstack/util/runner_utils.html +468 -0
  197. simstack/static/docs/_modules/simstack/util/safe_code_executor.html +510 -0
  198. simstack/static/docs/_modules/simstack/util/sample_config.html +414 -0
  199. simstack/static/docs/_modules/simstack/util/sensitive_content_filter.html +364 -0
  200. simstack/static/docs/_modules/simstack/util/setup_logging.html +365 -0
  201. simstack/static/docs/_modules/simstack/util/submit_to_watchdog.html +425 -0
  202. simstack/static/docs/_modules/simstack/util/toml_reader.html +482 -0
  203. simstack/static/docs/_modules/simstack/util/transform_file_name.html +357 -0
  204. simstack/static/docs/_modules/simstack/util/ui_tools.html +541 -0
  205. simstack/static/docs/_sources/architecture.rst.txt +25 -0
  206. simstack/static/docs/_sources/array_lists.rst.txt +60 -0
  207. simstack/static/docs/_sources/ci_pipeline.rst.txt +34 -0
  208. simstack/static/docs/_sources/data.rst.txt +175 -0
  209. simstack/static/docs/_sources/database_migrations.rst.txt +310 -0
  210. simstack/static/docs/_sources/datasets.rst.txt +93 -0
  211. simstack/static/docs/_sources/developer_guide.rst.txt +25 -0
  212. simstack/static/docs/_sources/docker_nodes.rst.txt +149 -0
  213. simstack/static/docs/_sources/files.rst.txt +159 -0
  214. simstack/static/docs/_sources/fire_and_forget_runner.rst.txt +67 -0
  215. simstack/static/docs/_sources/index.rst.txt +25 -0
  216. simstack/static/docs/_sources/installation.rst.txt +213 -0
  217. simstack/static/docs/_sources/lists.rst.txt +111 -0
  218. simstack/static/docs/_sources/mass_runner.rst.txt +91 -0
  219. simstack/static/docs/_sources/misc.rst.txt +106 -0
  220. simstack/static/docs/_sources/overview.rst.txt +91 -0
  221. simstack/static/docs/_sources/recovery.rst.txt +38 -0
  222. simstack/static/docs/_sources/resource_config.rst.txt +141 -0
  223. simstack/static/docs/_sources/simstack/modules/modules.rst.txt +7 -0
  224. simstack/static/docs/_sources/simstack/modules/simstack.core.rst.txt +205 -0
  225. simstack/static/docs/_sources/simstack/modules/simstack.core.services.rst.txt +109 -0
  226. simstack/static/docs/_sources/simstack/modules/simstack.methods.rst.txt +77 -0
  227. simstack/static/docs/_sources/simstack/modules/simstack.models.rst.txt +261 -0
  228. simstack/static/docs/_sources/simstack/modules/simstack.rst.txt +22 -0
  229. simstack/static/docs/_sources/simstack/modules/simstack.tables.rst.txt +45 -0
  230. simstack/static/docs/_sources/simstack/modules/simstack.util.rst.txt +381 -0
  231. simstack/static/docs/_sources/submit.rst.txt +141 -0
  232. simstack/static/docs/_sources/tables.rst.txt +50 -0
  233. simstack/static/docs/_sources/testing_complex_nodes.rst.txt +92 -0
  234. simstack/static/docs/_sources/ui_documentation.rst.txt +272 -0
  235. simstack/static/docs/_sources/workflows.rst.txt +446 -0
  236. simstack/static/docs/_static/base-stemmer.js +476 -0
  237. simstack/static/docs/_static/basic.css +906 -0
  238. simstack/static/docs/_static/cobalt2-theme.css +1 -0
  239. simstack/static/docs/_static/debug.css +69 -0
  240. simstack/static/docs/_static/docs-build-badge.css +51 -0
  241. simstack/static/docs/_static/docs-build-badge.js +51 -0
  242. simstack/static/docs/_static/doctools.js +150 -0
  243. simstack/static/docs/_static/documentation_options.js +13 -0
  244. simstack/static/docs/_static/english-stemmer.js +1066 -0
  245. simstack/static/docs/_static/file.png +0 -0
  246. simstack/static/docs/_static/language_data.js +13 -0
  247. simstack/static/docs/_static/minus.png +0 -0
  248. simstack/static/docs/_static/plus.png +0 -0
  249. simstack/static/docs/_static/pygments.css +250 -0
  250. simstack/static/docs/_static/scripts/furo-extensions.js +0 -0
  251. simstack/static/docs/_static/scripts/furo.js +3 -0
  252. simstack/static/docs/_static/scripts/furo.js.LICENSE.txt +7 -0
  253. simstack/static/docs/_static/scripts/furo.js.map +1 -0
  254. simstack/static/docs/_static/searchtools.js +693 -0
  255. simstack/static/docs/_static/skeleton.css +296 -0
  256. simstack/static/docs/_static/sphinx_highlight.js +159 -0
  257. simstack/static/docs/_static/styles/furo-extensions.css +2 -0
  258. simstack/static/docs/_static/styles/furo-extensions.css.map +1 -0
  259. simstack/static/docs/_static/styles/furo.css +2 -0
  260. simstack/static/docs/_static/styles/furo.css.map +1 -0
  261. simstack/static/docs/architecture.html +340 -0
  262. simstack/static/docs/array_lists.html +678 -0
  263. simstack/static/docs/ci_pipeline.html +385 -0
  264. simstack/static/docs/data.html +545 -0
  265. simstack/static/docs/database_migrations.html +652 -0
  266. simstack/static/docs/datasets.html +1268 -0
  267. simstack/static/docs/developer_guide.html +359 -0
  268. simstack/static/docs/docker_nodes.html +503 -0
  269. simstack/static/docs/files.html +718 -0
  270. simstack/static/docs/fire_and_forget_runner.html +641 -0
  271. simstack/static/docs/genindex.html +6390 -0
  272. simstack/static/docs/index.html +516 -0
  273. simstack/static/docs/installation.html +553 -0
  274. simstack/static/docs/lists.html +478 -0
  275. simstack/static/docs/mass_runner.html +481 -0
  276. simstack/static/docs/misc.html +454 -0
  277. simstack/static/docs/objects.inv +0 -0
  278. simstack/static/docs/overview.html +469 -0
  279. simstack/static/docs/py-modindex.html +1111 -0
  280. simstack/static/docs/recovery.html +371 -0
  281. simstack/static/docs/resource_config.html +590 -0
  282. simstack/static/docs/search.html +327 -0
  283. simstack/static/docs/searchindex.js +1 -0
  284. simstack/static/docs/simstack/modules/modules.html +489 -0
  285. simstack/static/docs/simstack/modules/simstack.core.html +1883 -0
  286. simstack/static/docs/simstack/modules/simstack.core.services.html +698 -0
  287. simstack/static/docs/simstack/modules/simstack.html +3060 -0
  288. simstack/static/docs/simstack/modules/simstack.methods.html +1150 -0
  289. simstack/static/docs/simstack/modules/simstack.models.html +13113 -0
  290. simstack/static/docs/simstack/modules/simstack.tables.html +635 -0
  291. simstack/static/docs/simstack/modules/simstack.util.html +3320 -0
  292. simstack/static/docs/submit.html +487 -0
  293. simstack/static/docs/tables.html +539 -0
  294. simstack/static/docs/testing_complex_nodes.html +458 -0
  295. simstack/static/docs/ui_documentation.html +788 -0
  296. simstack/static/docs/workflows.html +766 -0
  297. simstack/tables/__init__.py +0 -0
  298. simstack/tables/model_table.py +191 -0
  299. simstack/tables/node_children.py +219 -0
  300. simstack/tables/node_table.py +517 -0
  301. simstack/tables/table_builder.py +314 -0
  302. simstack/util/__init__.py +0 -0
  303. simstack/util/async_zip_utils.py +18 -0
  304. simstack/util/b64mixin.py +23 -0
  305. simstack/util/cleaned_json_schema.py +20 -0
  306. simstack/util/config_reader.py +179 -0
  307. simstack/util/custom_model_dump.py +75 -0
  308. simstack/util/database_information.py +160 -0
  309. simstack/util/db.py +1013 -0
  310. simstack/util/db_logger.py +117 -0
  311. simstack/util/default_from_dict.py +57 -0
  312. simstack/util/directory_iterator.py +216 -0
  313. simstack/util/docstring_parser.py +186 -0
  314. simstack/util/file_hashing.py +72 -0
  315. simstack/util/file_transfer_client.py +427 -0
  316. simstack/util/files.py +396 -0
  317. simstack/util/find_python_files.py +33 -0
  318. simstack/util/generate_ui_schema.py +80 -0
  319. simstack/util/generic_list_mixin.py +120 -0
  320. simstack/util/geometry_utils.py +118 -0
  321. simstack/util/get_module_path.py +20 -0
  322. simstack/util/git_pull.py +65 -0
  323. simstack/util/git_repository_status.py +95 -0
  324. simstack/util/import_module.py +63 -0
  325. simstack/util/importer.py +312 -0
  326. simstack/util/init_data_source.py +41 -0
  327. simstack/util/make_graph.py +107 -0
  328. simstack/util/make_table.py +837 -0
  329. simstack/util/mappings.py +41 -0
  330. simstack/util/minimal_route_finder.py +72 -0
  331. simstack/util/mock_db.py +144 -0
  332. simstack/util/module_path_checker.py +38 -0
  333. simstack/util/object_list_mixin.py +282 -0
  334. simstack/util/path_manager.py +223 -0
  335. simstack/util/project_root_finder.py +41 -0
  336. simstack/util/queue_watcher.py +262 -0
  337. simstack/util/resource_config.py +198 -0
  338. simstack/util/route_finder.py +74 -0
  339. simstack/util/runner_util.py +166 -0
  340. simstack/util/runner_utils.py +143 -0
  341. simstack/util/safe_code_executor.py +194 -0
  342. simstack/util/sample_config.py +98 -0
  343. simstack/util/sensitive_content_filter.py +48 -0
  344. simstack/util/setup_logging.py +49 -0
  345. simstack/util/submit_to_watchdog.py +106 -0
  346. simstack/util/toml_reader.py +142 -0
  347. simstack/util/transform_file_name.py +41 -0
  348. simstack/util/ui_tools.py +213 -0
  349. simstack-2.0.1.dist-info/METADATA +454 -0
  350. simstack-2.0.1.dist-info/RECORD +354 -0
  351. simstack-2.0.1.dist-info/WHEEL +5 -0
  352. simstack-2.0.1.dist-info/entry_points.txt +12 -0
  353. simstack-2.0.1.dist-info/licenses/LICENSE +21 -0
  354. simstack-2.0.1.dist-info/top_level.txt +1 -0
simstack/static/docs/_modules/simstack/models/files.html ADDED
@@ -0,0 +1,773 @@
1
+ <!doctype html>
2
+ <html class="no-js" lang="en" data-content_root="../../../">
3
+ <head><meta charset="utf-8">
4
+ <meta name="viewport" content="width=device-width,initial-scale=1">
5
+ <meta name="color-scheme" content="light dark"><link rel="index" title="Index" href="../../../genindex.html"><link rel="search" title="Search" href="../../../search.html">
6
+
7
+ <!-- Generated with Sphinx 9.1.0 and Furo 2025.12.19 -->
8
+ <title>simstack.models.files - SimStack II documentation</title>
9
+ <link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=d111a655" />
10
+ <link rel="stylesheet" type="text/css" href="../../../_static/styles/furo.css?v=7bdb33bb" />
11
+ <link rel="stylesheet" type="text/css" href="../../../_static/styles/furo-extensions.css?v=8dab3a3b" />
12
+ <link rel="stylesheet" type="text/css" href="../../../_static/cobalt2-theme.css?v=2d651046" />
13
+ <link rel="stylesheet" type="text/css" href="../../../_static/docs-build-badge.css?v=803faff8" />
14
+
15
+
16
+
17
+
18
+ <style>
19
+ body {
20
+ --color-code-background: #f2f2f2;
21
+ --color-code-foreground: #1e1e1e;
22
+
23
+ }
24
+ @media not print {
25
+ body[data-theme="dark"] {
26
+ --color-code-background: #202020;
27
+ --color-code-foreground: #d0d0d0;
28
+ --color-background-primary: #132738;
29
+ --color-background-secondary: #1e3a52;
30
+ --color-foreground-primary: #ffffff;
31
+ --color-foreground-secondary: #bac8d3;
32
+ --color-brand-primary: #22d5ff;
33
+ --color-brand-content: #0088ff;
34
+
35
+ }
36
+ @media (prefers-color-scheme: dark) {
37
+ body:not([data-theme="light"]) {
38
+ --color-code-background: #202020;
39
+ --color-code-foreground: #d0d0d0;
40
+ --color-background-primary: #132738;
41
+ --color-background-secondary: #1e3a52;
42
+ --color-foreground-primary: #ffffff;
43
+ --color-foreground-secondary: #bac8d3;
44
+ --color-brand-primary: #22d5ff;
45
+ --color-brand-content: #0088ff;
46
+
47
+ }
48
+ }
49
+ }
50
+ </style></head>
51
+ <body>
52
+
53
+ <script>
54
+ document.body.dataset.theme = localStorage.getItem("theme") || "auto";
55
+ </script>
56
+
57
+
58
+ <svg xmlns="http://www.w3.org/2000/svg" style="display: none;">
59
+ <symbol id="svg-toc" viewBox="0 0 24 24">
60
+ <title>Contents</title>
61
+ <svg stroke="currentColor" fill="currentColor" stroke-width="0" viewBox="0 0 1024 1024">
62
+ <path d="M408 442h480c4.4 0 8-3.6 8-8v-56c0-4.4-3.6-8-8-8H408c-4.4 0-8 3.6-8 8v56c0 4.4 3.6 8 8 8zm-8 204c0 4.4 3.6 8 8 8h480c4.4 0 8-3.6 8-8v-56c0-4.4-3.6-8-8-8H408c-4.4 0-8 3.6-8 8v56zm504-486H120c-4.4 0-8 3.6-8 8v56c0 4.4 3.6 8 8 8h784c4.4 0 8-3.6 8-8v-56c0-4.4-3.6-8-8-8zm0 632H120c-4.4 0-8 3.6-8 8v56c0 4.4 3.6 8 8 8h784c4.4 0 8-3.6 8-8v-56c0-4.4-3.6-8-8-8zM115.4 518.9L271.7 642c5.8 4.6 14.4.5 14.4-6.9V388.9c0-7.4-8.5-11.5-14.4-6.9L115.4 505.1a8.74 8.74 0 0 0 0 13.8z"/>
63
+ </svg>
64
+ </symbol>
65
+ <symbol id="svg-menu" viewBox="0 0 24 24">
66
+ <title>Menu</title>
67
+ <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
68
+ stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather-menu">
69
+ <line x1="3" y1="12" x2="21" y2="12"></line>
70
+ <line x1="3" y1="6" x2="21" y2="6"></line>
71
+ <line x1="3" y1="18" x2="21" y2="18"></line>
72
+ </svg>
73
+ </symbol>
74
+ <symbol id="svg-arrow-right" viewBox="0 0 24 24">
75
+ <title>Expand</title>
76
+ <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
77
+ stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather-chevron-right">
78
+ <polyline points="9 18 15 12 9 6"></polyline>
79
+ </svg>
80
+ </symbol>
81
+ <symbol id="svg-sun" viewBox="0 0 24 24">
82
+ <title>Light mode</title>
83
+ <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
84
+ stroke-width="1" stroke-linecap="round" stroke-linejoin="round" class="feather-sun">
85
+ <circle cx="12" cy="12" r="5"></circle>
86
+ <line x1="12" y1="1" x2="12" y2="3"></line>
87
+ <line x1="12" y1="21" x2="12" y2="23"></line>
88
+ <line x1="4.22" y1="4.22" x2="5.64" y2="5.64"></line>
89
+ <line x1="18.36" y1="18.36" x2="19.78" y2="19.78"></line>
90
+ <line x1="1" y1="12" x2="3" y2="12"></line>
91
+ <line x1="21" y1="12" x2="23" y2="12"></line>
92
+ <line x1="4.22" y1="19.78" x2="5.64" y2="18.36"></line>
93
+ <line x1="18.36" y1="5.64" x2="19.78" y2="4.22"></line>
94
+ </svg>
95
+ </symbol>
96
+ <symbol id="svg-moon" viewBox="0 0 24 24">
97
+ <title>Dark mode</title>
98
+ <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
99
+ stroke-width="1" stroke-linecap="round" stroke-linejoin="round" class="icon-tabler-moon">
100
+ <path stroke="none" d="M0 0h24v24H0z" fill="none" />
101
+ <path d="M12 3c.132 0 .263 0 .393 0a7.5 7.5 0 0 0 7.92 12.446a9 9 0 1 1 -8.313 -12.454z" />
102
+ </svg>
103
+ </symbol>
104
+ <symbol id="svg-sun-with-moon" viewBox="0 0 24 24">
105
+ <title>Auto light/dark, in light mode</title>
106
+ <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
107
+ stroke-width="1" stroke-linecap="round" stroke-linejoin="round"
108
+ class="icon-custom-derived-from-feather-sun-and-tabler-moon">
109
+ <path style="opacity: 50%" d="M 5.411 14.504 C 5.471 14.504 5.532 14.504 5.591 14.504 C 3.639 16.319 4.383 19.569 6.931 20.352 C 7.693 20.586 8.512 20.551 9.25 20.252 C 8.023 23.207 4.056 23.725 2.11 21.184 C 0.166 18.642 1.702 14.949 4.874 14.536 C 5.051 14.512 5.231 14.5 5.411 14.5 L 5.411 14.504 Z"/>
110
+ <line x1="14.5" y1="3.25" x2="14.5" y2="1.25"/>
111
+ <line x1="14.5" y1="15.85" x2="14.5" y2="17.85"/>
112
+ <line x1="10.044" y1="5.094" x2="8.63" y2="3.68"/>
113
+ <line x1="19" y1="14.05" x2="20.414" y2="15.464"/>
114
+ <line x1="8.2" y1="9.55" x2="6.2" y2="9.55"/>
115
+ <line x1="20.8" y1="9.55" x2="22.8" y2="9.55"/>
116
+ <line x1="10.044" y1="14.006" x2="8.63" y2="15.42"/>
117
+ <line x1="19" y1="5.05" x2="20.414" y2="3.636"/>
118
+ <circle cx="14.5" cy="9.55" r="3.6"/>
119
+ </svg>
120
+ </symbol>
121
+ <symbol id="svg-moon-with-sun" viewBox="0 0 24 24">
122
+ <title>Auto light/dark, in dark mode</title>
123
+ <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
124
+ stroke-width="1" stroke-linecap="round" stroke-linejoin="round"
125
+ class="icon-custom-derived-from-feather-sun-and-tabler-moon">
126
+ <path d="M 8.282 7.007 C 8.385 7.007 8.494 7.007 8.595 7.007 C 5.18 10.184 6.481 15.869 10.942 17.24 C 12.275 17.648 13.706 17.589 15 17.066 C 12.851 22.236 5.91 23.143 2.505 18.696 C -0.897 14.249 1.791 7.786 7.342 7.063 C 7.652 7.021 7.965 7 8.282 7 L 8.282 7.007 Z"/>
127
+ <line style="opacity: 50%" x1="18" y1="3.705" x2="18" y2="2.5"/>
128
+ <line style="opacity: 50%" x1="18" y1="11.295" x2="18" y2="12.5"/>
129
+ <line style="opacity: 50%" x1="15.316" y1="4.816" x2="14.464" y2="3.964"/>
130
+ <line style="opacity: 50%" x1="20.711" y1="10.212" x2="21.563" y2="11.063"/>
131
+ <line style="opacity: 50%" x1="14.205" y1="7.5" x2="13.001" y2="7.5"/>
132
+ <line style="opacity: 50%" x1="21.795" y1="7.5" x2="23" y2="7.5"/>
133
+ <line style="opacity: 50%" x1="15.316" y1="10.184" x2="14.464" y2="11.036"/>
134
+ <line style="opacity: 50%" x1="20.711" y1="4.789" x2="21.563" y2="3.937"/>
135
+ <circle style="opacity: 50%" cx="18" cy="7.5" r="2.169"/>
136
+ </svg>
137
+ </symbol>
138
+ <symbol id="svg-pencil" viewBox="0 0 24 24">
139
+ <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
140
+ stroke-width="1" stroke-linecap="round" stroke-linejoin="round" class="icon-tabler-pencil-code">
141
+ <path d="M4 20h4l10.5 -10.5a2.828 2.828 0 1 0 -4 -4l-10.5 10.5v4" />
142
+ <path d="M13.5 6.5l4 4" />
143
+ <path d="M20 21l2 -2l-2 -2" />
144
+ <path d="M17 17l-2 2l2 2" />
145
+ </svg>
146
+ </symbol>
147
+ <symbol id="svg-eye" viewBox="0 0 24 24">
148
+ <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
149
+ stroke-width="1" stroke-linecap="round" stroke-linejoin="round" class="icon-tabler-eye-code">
150
+ <path stroke="none" d="M0 0h24v24H0z" fill="none" />
151
+ <path d="M10 12a2 2 0 1 0 4 0a2 2 0 0 0 -4 0" />
152
+ <path
153
+ d="M11.11 17.958c-3.209 -.307 -5.91 -2.293 -8.11 -5.958c2.4 -4 5.4 -6 9 -6c3.6 0 6.6 2 9 6c-.21 .352 -.427 .688 -.647 1.008" />
154
+ <path d="M20 21l2 -2l-2 -2" />
155
+ <path d="M17 17l-2 2l2 2" />
156
+ </svg>
157
+ </symbol>
158
+ </svg>
159
+
160
+ <input type="checkbox" class="sidebar-toggle" name="__navigation" id="__navigation" aria-label="Toggle site navigation sidebar">
161
+ <input type="checkbox" class="sidebar-toggle" name="__toc" id="__toc" aria-label="Toggle table of contents sidebar">
162
+ <label class="overlay sidebar-overlay" for="__navigation"></label>
163
+ <label class="overlay toc-overlay" for="__toc"></label>
164
+
165
+ <a class="skip-to-content muted-link" href="#furo-main-content">Skip to content</a>
166
+
167
+
168
+
169
+ <div class="page">
170
+ <header class="mobile-header">
171
+ <div class="header-left">
172
+ <label class="nav-overlay-icon" for="__navigation">
173
+ <span class="icon"><svg><use href="#svg-menu"></use></svg></span>
174
+ </label>
175
+ </div>
176
+ <div class="header-center">
177
+ <a href="../../../index.html"><div class="brand">SimStack II documentation</div></a>
178
+ </div>
179
+ <div class="header-right">
180
+ <div class="theme-toggle-container theme-toggle-header">
181
+ <button class="theme-toggle" aria-label="Toggle Light / Dark / Auto color theme">
182
+ <svg class="theme-icon-when-auto-light"><use href="#svg-sun-with-moon"></use></svg>
183
+ <svg class="theme-icon-when-auto-dark"><use href="#svg-moon-with-sun"></use></svg>
184
+ <svg class="theme-icon-when-dark"><use href="#svg-moon"></use></svg>
185
+ <svg class="theme-icon-when-light"><use href="#svg-sun"></use></svg>
186
+ </button>
187
+ </div>
188
+ <label class="toc-overlay-icon toc-header-icon no-toc" for="__toc">
189
+ <span class="icon"><svg><use href="#svg-toc"></use></svg></span>
190
+ </label>
191
+ </div>
192
+ </header>
193
+ <aside class="sidebar-drawer">
194
+ <div class="sidebar-container">
195
+
196
+ <div class="sidebar-sticky"><a class="sidebar-brand" href="../../../index.html">
197
+
198
+
199
+ </a><form class="sidebar-search-container" method="get" action="../../../search.html" role="search">
200
+ <input class="sidebar-search" placeholder="Search" name="q" aria-label="Search">
201
+ <input type="hidden" name="check_keywords" value="yes">
202
+ <input type="hidden" name="area" value="default">
203
+ </form>
204
+ <div id="searchbox"></div><div class="sidebar-scroll"><div class="sidebar-tree">
205
+ <p class="caption" role="heading"><span class="caption-text">Contents:</span></p>
206
+ <ul>
207
+ <li class="toctree-l1"><a class="reference internal" href="../../../overview.html">Overview</a></li>
208
+ <li class="toctree-l1"><a class="reference internal" href="../../../workflows.html">Workflows</a></li>
209
+ <li class="toctree-l1"><a class="reference internal" href="../../../ui_documentation.html">Graphical User Interface</a></li>
210
+ <li class="toctree-l1 has-children"><a class="reference internal" href="../../../developer_guide.html">Developer Guide</a><input aria-label="Toggle navigation of Developer Guide" class="toctree-checkbox" id="toctree-checkbox-1" name="toctree-checkbox-1" role="switch" type="checkbox"/><label for="toctree-checkbox-1"><span class="icon"><svg><use href="#svg-arrow-right"></use></svg></span></label><ul>
211
+ <li class="toctree-l2"><a class="reference internal" href="../../../data.html">Persisting and Visualizing Data</a></li>
212
+ <li class="toctree-l2"><a class="reference internal" href="../../../datasets.html">Datasets</a></li>
213
+ <li class="toctree-l2"><a class="reference internal" href="../../../lists.html">List Mixins</a></li>
214
+ <li class="toctree-l2"><a class="reference internal" href="../../../array_lists.html">Array Lists</a></li>
215
+ <li class="toctree-l2"><a class="reference internal" href="../../../tables.html">Tables</a></li>
216
+ <li class="toctree-l2"><a class="reference internal" href="../../../files.html">File Management</a></li>
217
+ <li class="toctree-l2"><a class="reference internal" href="../../../resource_config.html">Program Execution</a></li>
218
+ <li class="toctree-l2"><a class="reference internal" href="../../../mass_runner.html">MassRunner</a></li>
219
+ <li class="toctree-l2"><a class="reference internal" href="../../../fire_and_forget_runner.html">FireAndForgetRunner</a></li>
220
+ <li class="toctree-l2"><a class="reference internal" href="../../../docker_nodes.html">Creating Docker Containers for SimStack Nodes</a></li>
221
+ <li class="toctree-l2"><a class="reference internal" href="../../../testing_complex_nodes.html">Testing Complex Nodes</a></li>
222
+ <li class="toctree-l2"><a class="reference internal" href="../../../database_migrations.html">Database Migrations</a></li>
223
+ <li class="toctree-l2"><a class="reference internal" href="../../../ci_pipeline.html">CI Pipeline and Documentation Artifacts</a></li>
224
+ <li class="toctree-l2"><a class="reference internal" href="../../../misc.html">Miscellaneous</a></li>
225
+ </ul>
226
+ </li>
227
+ <li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Simstack II Setup</a></li>
228
+ <li class="toctree-l1 has-children"><a class="reference internal" href="../../../simstack/modules/modules.html">simstack</a><input aria-label="Toggle navigation of simstack" class="toctree-checkbox" id="toctree-checkbox-2" name="toctree-checkbox-2" role="switch" type="checkbox"/><label for="toctree-checkbox-2"><span class="icon"><svg><use href="#svg-arrow-right"></use></svg></span></label><ul>
229
+ <li class="toctree-l2 has-children"><a class="reference internal" href="../../../simstack/modules/simstack.html">simstack package</a><input aria-label="Toggle navigation of simstack package" class="toctree-checkbox" id="toctree-checkbox-3" name="toctree-checkbox-3" role="switch" type="checkbox"/><label for="toctree-checkbox-3"><span class="icon"><svg><use href="#svg-arrow-right"></use></svg></span></label><ul>
230
+ <li class="toctree-l3 has-children"><a class="reference internal" href="../../../simstack/modules/simstack.core.html">simstack.core package</a><input aria-label="Toggle navigation of simstack.core package" class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" role="switch" type="checkbox"/><label for="toctree-checkbox-4"><span class="icon"><svg><use href="#svg-arrow-right"></use></svg></span></label><ul>
231
+ <li class="toctree-l4"><a class="reference internal" href="../../../simstack/modules/simstack.core.services.html">simstack.core.services package</a></li>
232
+ </ul>
233
+ </li>
234
+ <li class="toctree-l3"><a class="reference internal" href="../../../simstack/modules/simstack.methods.html">simstack.methods package</a></li>
235
+ <li class="toctree-l3"><a class="reference internal" href="../../../simstack/modules/simstack.models.html">simstack.models package</a></li>
236
+ <li class="toctree-l3"><a class="reference internal" href="../../../simstack/modules/simstack.tables.html">simstack.tables package</a></li>
237
+ <li class="toctree-l3"><a class="reference internal" href="../../../simstack/modules/simstack.util.html">simstack.util package</a></li>
238
+ </ul>
239
+ </li>
240
+ </ul>
241
+ </li>
242
+ </ul>
243
+
244
+ </div>
245
+ </div>
246
+
247
+ </div>
248
+
249
+ </div>
250
+ </aside>
251
+ <div class="main">
252
+ <div class="content">
253
+ <div class="article-container">
254
+ <a href="#" class="back-to-top muted-link">
255
+ <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24">
256
+ <path d="M13 20h-2V8l-5.5 5.5-1.42-1.42L12 4.16l7.92 7.92-1.42 1.42L13 8v12z"></path>
257
+ </svg>
258
+ <span>Back to top</span>
259
+ </a>
260
+ <div class="content-icon-container">
261
+ <div class="theme-toggle-container theme-toggle-content">
262
+ <button class="theme-toggle" aria-label="Toggle Light / Dark / Auto color theme">
263
+ <svg class="theme-icon-when-auto-light"><use href="#svg-sun-with-moon"></use></svg>
264
+ <svg class="theme-icon-when-auto-dark"><use href="#svg-moon-with-sun"></use></svg>
265
+ <svg class="theme-icon-when-dark"><use href="#svg-moon"></use></svg>
266
+ <svg class="theme-icon-when-light"><use href="#svg-sun"></use></svg>
267
+ </button>
268
+ </div>
269
+ <label class="toc-overlay-icon toc-content-icon no-toc" for="__toc">
270
+ <span class="icon"><svg><use href="#svg-toc"></use></svg></span>
271
+ </label>
272
+ </div>
273
+ <article role="main" id="furo-main-content">
274
+ <h1>Source code for simstack.models.files</h1><div class="highlight"><pre>
275
+ <span></span><span class="kn">from</span><span class="w"> </span><span class="nn">__future__</span><span class="w"> </span><span class="kn">import</span> <span class="n">annotations</span>
276
+
277
+ <span class="kn">import</span><span class="w"> </span><span class="nn">logging</span>
278
+ <span class="kn">import</span><span class="w"> </span><span class="nn">os</span>
279
+ <span class="kn">import</span><span class="w"> </span><span class="nn">zlib</span>
280
+ <span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span>
281
+ <span class="kn">from</span><span class="w"> </span><span class="nn">pathlib</span><span class="w"> </span><span class="kn">import</span> <span class="n">Path</span>
282
+ <span class="kn">from</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="kn">import</span> <span class="n">List</span><span class="p">,</span> <span class="n">Optional</span><span class="p">,</span> <span class="n">Union</span><span class="p">,</span> <span class="n">Dict</span><span class="p">,</span> <span class="n">Any</span>
283
+
284
+ <span class="kn">from</span><span class="w"> </span><span class="nn">odmantic</span><span class="w"> </span><span class="kn">import</span> <span class="n">Model</span><span class="p">,</span> <span class="n">Field</span><span class="p">,</span> <span class="n">ObjectId</span><span class="p">,</span> <span class="n">Reference</span>
285
+
286
+ <span class="kn">from</span><span class="w"> </span><span class="nn">simstack.models</span><span class="w"> </span><span class="kn">import</span> <span class="n">simstack_model</span>
287
+ <span class="kn">from</span><span class="w"> </span><span class="nn">simstack.models.file_instance</span><span class="w"> </span><span class="kn">import</span> <span class="n">FileInstance</span>
288
+ <span class="kn">from</span><span class="w"> </span><span class="nn">simstack.models.parameters</span><span class="w"> </span><span class="kn">import</span> <span class="n">Resource</span>
289
+ <span class="kn">from</span><span class="w"> </span><span class="nn">simstack.util.file_transfer_client</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
290
+ <span class="n">FileTransferClient</span><span class="p">,</span>
291
+ <span class="n">first_available_remote_location</span><span class="p">,</span>
292
+ <span class="n">path_for_file_instance</span><span class="p">,</span>
293
+ <span class="n">transfer_resource_name</span><span class="p">,</span>
294
+ <span class="p">)</span>
295
+ <span class="kn">from</span><span class="w"> </span><span class="nn">simstack.util.file_hashing</span><span class="w"> </span><span class="kn">import</span> <span class="n">hash_file</span><span class="p">,</span> <span class="n">hash_string</span>
296
+
297
+ <span class="n">logger</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="vm">__name__</span><span class="p">)</span>
298
+
299
+ <span class="c1"># MongoDB has a 16MB document size limit</span>
300
+ <span class="n">MONGODB_MAX_DOCUMENT_SIZE</span> <span class="o">=</span> <span class="mi">16</span> <span class="o">*</span> <span class="mi">1024</span> <span class="o">*</span> <span class="mi">1024</span> <span class="c1"># 16 MB in bytes</span>
301
+
302
+ <span class="c1"># TODO: its a mystery to me why we should have files with no name or unknown size, both values are actually used in __str__</span>
303
+
304
+ <div class="viewcode-block" id="FileStack">
305
+ <a class="viewcode-back" href="../../../simstack/modules/simstack.models.html#simstack.models.files.FileStack">[docs]</a>
306
+ <span class="nd">@simstack_model</span>
307
+ <span class="k">class</span><span class="w"> </span><span class="nc">FileStack</span><span class="p">(</span><span class="n">Model</span><span class="p">):</span>
308
+ <span class="n">name</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span><span class="n">description</span><span class="o">=</span><span class="s2">&quot;Name of the file&quot;</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
309
+ <span class="n">size</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span><span class="n">description</span><span class="o">=</span><span class="s2">&quot;Size of the file in bytes&quot;</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
310
+ <span class="n">is_hashable</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
311
+ <span class="n">default</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">description</span><span class="o">=</span><span class="s2">&quot;Whether the file stack is hashable&quot;</span>
312
+ <span class="p">)</span>
313
+ <span class="nb">hash</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
314
+ <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">description</span><span class="o">=</span><span class="s2">&quot;Hash of the file stack content&quot;</span><span class="p">,</span> <span class="n">index</span><span class="o">=</span><span class="kc">True</span>
315
+ <span class="p">)</span>
316
+ <span class="n">in_memory</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
317
+ <span class="n">default</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">description</span><span class="o">=</span><span class="s2">&quot;Whether the file stack is in memory&quot;</span>
318
+ <span class="p">)</span>
319
+ <span class="n">content</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bytes</span><span class="p">]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
320
+ <span class="n">description</span><span class="o">=</span><span class="s2">&quot;Compressed file content&quot;</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span>
321
+ <span class="p">)</span>
322
+
323
+ <span class="n">locations</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">FileInstance</span><span class="p">]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
324
+ <span class="n">default_factory</span><span class="o">=</span><span class="nb">list</span><span class="p">,</span> <span class="n">description</span><span class="o">=</span><span class="s2">&quot;List of file locations&quot;</span>
325
+ <span class="p">)</span>
326
+
327
+ <span class="k">def</span><span class="w"> </span><span class="fm">__str__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
328
+ <span class="k">return</span> <span class="sa">f</span><span class="s2">&quot;FileStack(name=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="si">}</span><span class="s2">, size=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">size</span><span class="si">}</span><span class="s2">, is_hashable=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">is_hashable</span><span class="si">}</span><span class="s2">, in_memory=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">in_memory</span><span class="si">}</span><span class="s2">, locations=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">locations</span><span class="si">}</span><span class="s2">)&quot;</span>
329
+
330
+ <span class="k">def</span><span class="w"> </span><span class="fm">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
331
+ <span class="k">return</span> <span class="sa">f</span><span class="s2">&quot;FileStack(name=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="si">}</span><span class="s2">, size=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">size</span><span class="si">}</span><span class="s2">, is_hashable=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">is_hashable</span><span class="si">}</span><span class="s2">, in_memory=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">in_memory</span><span class="si">}</span><span class="s2">, locations=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">locations</span><span class="si">}</span><span class="s2">)&quot;</span>
332
+
333
+ <div class="viewcode-block" id="FileStack.custom_model_dump">
334
+ <a class="viewcode-back" href="../../../simstack/modules/simstack.models.html#simstack.models.files.FileStack.custom_model_dump">[docs]</a>
335
+ <span class="k">async</span> <span class="k">def</span><span class="w"> </span><span class="nf">custom_model_dump</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="n">Any</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]:</span>
336
+ <span class="n">dumped_data</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">model_dump</span><span class="p">()</span>
337
+ <span class="k">del</span> <span class="n">dumped_data</span><span class="p">[</span><span class="s2">&quot;content&quot;</span><span class="p">]</span> <span class="c1"># Exclude content from the dumped data</span>
338
+ <span class="k">return</span> <span class="n">dumped_data</span></div>
339
+
340
+
341
+ <div class="viewcode-block" id="FileStack.ui_base_schema">
342
+ <a class="viewcode-back" href="../../../simstack/modules/simstack.models.html#simstack.models.files.FileStack.ui_base_schema">[docs]</a>
343
+ <span class="nd">@classmethod</span>
344
+ <span class="k">def</span><span class="w"> </span><span class="nf">ui_base_schema</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="n">Any</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]:</span>
345
+ <span class="c1"># TODO get the model programatically</span>
346
+ <span class="k">return</span> <span class="p">{</span>
347
+ <span class="s2">&quot;ui:field&quot;</span><span class="p">:</span> <span class="s2">&quot;FileField&quot;</span><span class="p">,</span>
348
+ <span class="s2">&quot;ui:options&quot;</span><span class="p">:</span> <span class="p">{</span>
349
+ <span class="s2">&quot;model&quot;</span><span class="p">:</span> <span class="s2">&quot;simstack.models.files.FileStack&quot;</span><span class="p">,</span>
350
+ <span class="p">},</span>
351
+ <span class="p">}</span></div>
352
+
353
+
354
+ <div class="viewcode-block" id="FileStack.from_string">
355
+ <a class="viewcode-back" href="../../../simstack/modules/simstack.models.html#simstack.models.files.FileStack.from_string">[docs]</a>
356
+ <span class="nd">@classmethod</span>
357
+ <span class="k">def</span><span class="w"> </span><span class="nf">from_string</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">data_string</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">file_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">FileStack</span><span class="p">:</span>
358
+ <span class="n">content</span> <span class="o">=</span> <span class="n">zlib</span><span class="o">.</span><span class="n">compress</span><span class="p">(</span><span class="n">data_string</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s2">&quot;utf-8&quot;</span><span class="p">))</span>
359
+ <span class="n">file_hash</span> <span class="o">=</span> <span class="n">hash_string</span><span class="p">(</span><span class="n">data_string</span><span class="p">)</span>
360
+ <span class="n">size</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">content</span><span class="p">)</span>
361
+
362
+ <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">content</span><span class="p">)</span> <span class="o">&gt;</span> <span class="n">MONGODB_MAX_DOCUMENT_SIZE</span><span class="p">:</span>
363
+ <span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span>
364
+ <span class="sa">f</span><span class="s2">&quot;Compressed content size </span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="n">content</span><span class="p">)</span><span class="si">}</span><span class="s2"> bytes exceeds MongoDB limit of </span><span class="si">{</span><span class="n">MONGODB_MAX_DOCUMENT_SIZE</span><span class="si">}</span><span class="s2"> bytes for file </span><span class="si">{</span><span class="n">file_name</span><span class="si">}</span><span class="s2">&quot;</span>
365
+ <span class="p">)</span>
366
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
367
+ <span class="sa">f</span><span class="s2">&quot;Compressed content size </span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="n">content</span><span class="p">)</span><span class="si">}</span><span class="s2"> bytes exceeds MongoDB document size limit of </span><span class="si">{</span><span class="n">MONGODB_MAX_DOCUMENT_SIZE</span><span class="si">}</span><span class="s2"> bytes&quot;</span>
368
+ <span class="p">)</span>
369
+
370
+ <span class="n">file_stack</span> <span class="o">=</span> <span class="bp">cls</span><span class="p">(</span>
371
+ <span class="n">name</span><span class="o">=</span><span class="n">file_name</span><span class="p">,</span>
372
+ <span class="n">in_memory</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
373
+ <span class="n">content</span><span class="o">=</span><span class="n">content</span><span class="p">,</span>
374
+ <span class="n">is_hashable</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
375
+ <span class="n">size</span><span class="o">=</span><span class="n">size</span><span class="p">,</span>
376
+ <span class="nb">hash</span><span class="o">=</span><span class="n">file_hash</span><span class="p">,</span>
377
+ <span class="p">)</span>
378
+ <span class="k">return</span> <span class="n">file_stack</span></div>
379
+
380
+
381
+ <div class="viewcode-block" id="FileStack.from_local_file">
382
+ <a class="viewcode-back" href="../../../simstack/modules/simstack.models.html#simstack.models.files.FileStack.from_local_file">[docs]</a>
383
+ <span class="nd">@classmethod</span>
384
+ <span class="k">def</span><span class="w"> </span><span class="nf">from_local_file</span><span class="p">(</span>
385
+ <span class="bp">cls</span><span class="p">,</span>
386
+ <span class="n">path</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">Path</span><span class="p">,</span> <span class="nb">str</span><span class="p">],</span>
387
+ <span class="n">is_hashable</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
388
+ <span class="n">in_memory</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
389
+ <span class="n">secure_source</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
390
+ <span class="n">task_id</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span><span class="p">,</span>
391
+ <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">FileStack</span><span class="p">:</span>
392
+ <span class="w"> </span><span class="sd">&quot;&quot;&quot;</span>
393
+ <span class="sd"> Creates a FileStack object from a local file path.</span>
394
+
395
+ <span class="sd"> :param task_id: task_id of the task that created the file stack, used for logging and tracking</span>
396
+ <span class="sd"> :type task_id: str</span>
397
+ <span class="sd"> :param secure_source: specifies if the source is secure (already in a directory generated within Simstack II)</span>
398
+ <span class="sd"> :type secure_source: bool</span>
399
+ <span class="sd"> :param path: The path to the local file. Can be provided as a string or Path object.</span>
400
+ <span class="sd"> :type path: Union[Path, str]</span>
401
+ <span class="sd"> :param is_hashable: A flag indicating whether the file hash needs to be calculated.</span>
402
+ <span class="sd"> :type is_hashable: bool</span>
403
+ <span class="sd"> :param in_memory: Whether to store the compressed file content in memory. Defaults to True.</span>
404
+ <span class="sd"> :type in_memory: bool</span>
405
+
406
+ <span class="sd"> :return: A FileStack object containing FileInstances for the file.</span>
407
+ <span class="sd"> :rtype: FileStack</span>
408
+ <span class="sd"> &quot;&quot;&quot;</span>
409
+ <span class="c1"># TODO make a second version where a unique directory is already there</span>
410
+ <span class="n">source_path</span> <span class="o">=</span> <span class="n">path</span> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">Path</span><span class="p">)</span> <span class="k">else</span> <span class="n">Path</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
411
+
412
+ <span class="c1"># Check if the source exists</span>
413
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">source_path</span><span class="o">.</span><span class="n">exists</span><span class="p">():</span>
414
+ <span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Source file not found: </span><span class="si">{</span><span class="n">source_path</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
415
+ <span class="k">raise</span> <span class="ne">FileNotFoundError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Source file not found: </span><span class="si">{</span><span class="n">source_path</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
416
+
417
+ <span class="c1"># Check if it&#39;s a file (not a directory)</span>
418
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">source_path</span><span class="o">.</span><span class="n">is_file</span><span class="p">():</span>
419
+ <span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Source file </span><span class="si">{</span><span class="n">path</span><span class="si">}</span><span class="s2"> is not a file&quot;</span><span class="p">)</span>
420
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Source file is not a file: </span><span class="si">{</span><span class="n">path</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
421
+
422
+ <span class="c1"># Check read permission using os.access</span>
423
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">access</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">os</span><span class="o">.</span><span class="n">R_OK</span><span class="p">):</span>
424
+ <span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;No read permission for file: </span><span class="si">{</span><span class="n">path</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
425
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;No read permission for file: </span><span class="si">{</span><span class="n">path</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
426
+
427
+ <span class="n">content</span> <span class="o">=</span> <span class="kc">None</span>
428
+ <span class="n">file_hash</span> <span class="o">=</span> <span class="n">hash_file</span><span class="p">(</span><span class="n">source_path</span><span class="p">)</span> <span class="k">if</span> <span class="n">is_hashable</span> <span class="k">else</span> <span class="kc">None</span>
429
+ <span class="n">name</span> <span class="o">=</span> <span class="n">source_path</span><span class="o">.</span><span class="n">name</span>
430
+ <span class="n">size</span> <span class="o">=</span> <span class="n">source_path</span><span class="o">.</span><span class="n">stat</span><span class="p">()</span><span class="o">.</span><span class="n">st_size</span> <span class="k">if</span> <span class="n">source_path</span><span class="o">.</span><span class="n">exists</span><span class="p">()</span> <span class="k">else</span> <span class="kc">None</span>
431
+
432
+ <span class="k">if</span> <span class="n">in_memory</span><span class="p">:</span>
433
+ <span class="k">try</span><span class="p">:</span>
434
+ <span class="c1"># Read the file content</span>
435
+ <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">source_path</span><span class="p">,</span> <span class="s2">&quot;rb&quot;</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
436
+ <span class="n">file_content</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">read</span><span class="p">()</span>
437
+ <span class="c1"># Compress the content using zlib</span>
438
+ <span class="n">content</span> <span class="o">=</span> <span class="n">zlib</span><span class="o">.</span><span class="n">compress</span><span class="p">(</span><span class="n">file_content</span><span class="p">)</span>
439
+ <span class="k">if</span> <span class="n">task_id</span> <span class="o">==</span> <span class="s2">&quot;&quot;</span><span class="p">:</span>
440
+ <span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
441
+ <span class="sa">f</span><span class="s2">&quot;Compressed file </span><span class="si">{</span><span class="n">source_path</span><span class="si">}</span><span class="s2"> from </span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="n">file_content</span><span class="p">)</span><span class="si">}</span><span class="s2"> bytes to </span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="n">content</span><span class="p">)</span><span class="si">}</span><span class="s2"> bytes&quot;</span>
442
+ <span class="p">)</span>
443
+ <span class="k">else</span><span class="p">:</span>
444
+ <span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
445
+ <span class="sa">f</span><span class="s2">&quot;task_id: </span><span class="si">{</span><span class="n">task_id</span><span class="si">}</span><span class="s2"> Compressed file </span><span class="si">{</span><span class="n">source_path</span><span class="si">}</span><span class="s2"> from </span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="n">file_content</span><span class="p">)</span><span class="si">}</span><span class="s2"> bytes to </span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="n">content</span><span class="p">)</span><span class="si">}</span><span class="s2"> bytes&quot;</span>
446
+ <span class="p">)</span>
447
+ <span class="c1"># Check if compressed content exceeds MongoDB document size limit</span>
448
+ <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">content</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mf">0.9</span> <span class="o">*</span> <span class="n">MONGODB_MAX_DOCUMENT_SIZE</span><span class="p">:</span>
449
+ <span class="k">if</span> <span class="n">task_id</span> <span class="o">==</span> <span class="s2">&quot;&quot;</span><span class="p">:</span>
450
+ <span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span>
451
+ <span class="sa">f</span><span class="s2">&quot;Compressed content size </span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="n">content</span><span class="p">)</span><span class="si">}</span><span class="s2"> bytes exceeds MongoDB limit of </span><span class="si">{</span><span class="n">MONGODB_MAX_DOCUMENT_SIZE</span><span class="si">}</span><span class="s2"> bytes for file </span><span class="si">{</span><span class="n">source_path</span><span class="si">}</span><span class="s2">&quot;</span>
452
+ <span class="p">)</span>
453
+ <span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span>
454
+ <span class="sa">f</span><span class="s2">&quot;Setting in_memory to False for file </span><span class="si">{</span><span class="n">source_path</span><span class="si">}</span><span class="s2"> and clear content&quot;</span>
455
+ <span class="p">)</span>
456
+ <span class="k">else</span><span class="p">:</span>
457
+ <span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span>
458
+ <span class="sa">f</span><span class="s2">&quot;task_id: </span><span class="si">{</span><span class="n">task_id</span><span class="si">}</span><span class="s2"> Compressed content size </span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="n">content</span><span class="p">)</span><span class="si">}</span><span class="s2"> bytes exceeds MongoDB limit of </span><span class="si">{</span><span class="n">MONGODB_MAX_DOCUMENT_SIZE</span><span class="si">}</span><span class="s2"> bytes for file </span><span class="si">{</span><span class="n">source_path</span><span class="si">}</span><span class="s2">&quot;</span>
459
+ <span class="p">)</span>
460
+ <span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span>
461
+ <span class="sa">f</span><span class="s2">&quot;task_id: </span><span class="si">{</span><span class="n">task_id</span><span class="si">}</span><span class="s2"> Setting in_memory to False for file </span><span class="si">{</span><span class="n">source_path</span><span class="si">}</span><span class="s2"> and clear content&quot;</span>
462
+ <span class="p">)</span>
463
+ <span class="n">in_memory</span> <span class="o">=</span> <span class="kc">False</span>
464
+ <span class="n">content</span> <span class="o">=</span> <span class="kc">None</span>
465
+ <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
466
+ <span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Failed to compress file </span><span class="si">{</span><span class="n">source_path</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
467
+ <span class="k">raise</span>
468
+
469
+ <span class="n">file_stack</span> <span class="o">=</span> <span class="bp">cls</span><span class="p">(</span>
470
+ <span class="n">name</span><span class="o">=</span><span class="n">name</span><span class="p">,</span>
471
+ <span class="n">in_memory</span><span class="o">=</span><span class="n">in_memory</span><span class="p">,</span>
472
+ <span class="n">content</span><span class="o">=</span><span class="n">content</span><span class="p">,</span>
473
+ <span class="n">is_hashable</span><span class="o">=</span><span class="n">is_hashable</span><span class="p">,</span>
474
+ <span class="n">size</span><span class="o">=</span><span class="n">size</span><span class="p">,</span>
475
+ <span class="nb">hash</span><span class="o">=</span><span class="n">file_hash</span><span class="p">,</span>
476
+ <span class="p">)</span>
477
+
478
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">in_memory</span><span class="p">:</span>
479
+ <span class="n">location</span> <span class="o">=</span> <span class="n">FileInstance</span><span class="o">.</span><span class="n">from_local_file</span><span class="p">(</span>
480
+ <span class="n">path</span><span class="o">=</span><span class="n">path</span><span class="p">,</span>
481
+ <span class="n">file_stack_id</span><span class="o">=</span><span class="n">file_stack</span><span class="o">.</span><span class="n">id</span><span class="p">,</span>
482
+ <span class="n">make_copy</span><span class="o">=</span><span class="ow">not</span> <span class="n">secure_source</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">in_memory</span><span class="p">,</span>
483
+ <span class="p">)</span>
484
+ <span class="n">file_stack</span><span class="o">.</span><span class="n">locations</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">location</span><span class="p">)</span>
485
+
486
+ <span class="k">return</span> <span class="n">file_stack</span></div>
487
+
488
+
489
+ <div class="viewcode-block" id="FileStack.complex_hash">
490
+ <a class="viewcode-back" href="../../../simstack/modules/simstack.models.html#simstack.models.files.FileStack.complex_hash">[docs]</a>
491
+ <span class="k">def</span><span class="w"> </span><span class="nf">complex_hash</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
492
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_hashable</span><span class="p">:</span>
493
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">hash</span><span class="p">:</span>
494
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">hash</span>
495
+ <span class="k">else</span><span class="p">:</span>
496
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;FileStack is hashable but hash is not set.&quot;</span><span class="p">)</span>
497
+ <span class="c1"># elif self.in_memory and self.content:</span>
498
+ <span class="c1"># # If the content is in memory, hash the compressed content</span>
499
+ <span class="c1"># return complex_hash_function(zlib.decompress(self.content))</span>
500
+ <span class="c1"># else:</span>
501
+ <span class="c1"># temp_dir = Path(tempfile.mkdtemp())</span>
502
+ <span class="c1"># local_file = self.get(None, local_dir=temp_dir)</span>
503
+ <span class="c1"># return complex_hash_function(local_file.read_bytes())</span>
504
+ <span class="k">else</span><span class="p">:</span>
505
+ <span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
506
+ <span class="sa">f</span><span class="s2">&quot;FileStack </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">id</span><span class="si">}</span><span class="s2"> is not hashable, returning unique hash.&quot;</span>
507
+ <span class="p">)</span>
508
+ <span class="k">return</span> <span class="nb">str</span><span class="p">(</span><span class="n">ObjectId</span><span class="p">())</span></div>
509
+
510
+
511
+ <div class="viewcode-block" id="FileStack.append">
512
+ <a class="viewcode-back" href="../../../simstack/modules/simstack.models.html#simstack.models.files.FileStack.append">[docs]</a>
513
+ <span class="k">def</span><span class="w"> </span><span class="nf">append</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">file_instance</span><span class="p">:</span> <span class="n">FileInstance</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
514
+ <span class="w"> </span><span class="sd">&quot;&quot;&quot;</span>
515
+ <span class="sd"> Appends a FileInstance to the file stack.</span>
516
+
517
+ <span class="sd"> :param file_instance: The FileInstance to append.</span>
518
+ <span class="sd"> :type file_instance: FileInstance</span>
519
+ <span class="sd"> &quot;&quot;&quot;</span>
520
+ <span class="bp">self</span><span class="o">.</span><span class="n">locations</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">file_instance</span><span class="p">)</span></div>
521
+
522
+
523
+ <div class="viewcode-block" id="FileStack.get">
524
+ <a class="viewcode-back" href="../../../simstack/modules/simstack.models.html#simstack.models.files.FileStack.get">[docs]</a>
525
+ <span class="k">def</span><span class="w"> </span><span class="nf">get</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">local_dir</span><span class="p">:</span> <span class="n">Path</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Path</span><span class="p">:</span>
526
+ <span class="w"> </span><span class="sd">&quot;&quot;&quot;</span>
527
+ <span class="sd"> Copies the file stack to a local directory. This is the version to be used in applications</span>
528
+
529
+ <span class="sd"> :param local_dir: The local directory to copy the file stack to.</span>
530
+ <span class="sd"> :type local_dir: Path</span>
531
+ <span class="sd"> &quot;&quot;&quot;</span>
532
+ <span class="kn">from</span><span class="w"> </span><span class="nn">simstack.core.context</span><span class="w"> </span><span class="kn">import</span> <span class="n">context</span>
533
+
534
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_raw</span><span class="p">(</span><span class="n">context</span><span class="o">.</span><span class="n">config</span><span class="o">.</span><span class="n">resource</span><span class="p">,</span> <span class="n">local_dir</span><span class="p">)</span></div>
535
+
536
+
537
+ <div class="viewcode-block" id="FileStack.get_raw">
538
+ <a class="viewcode-back" href="../../../simstack/modules/simstack.models.html#simstack.models.files.FileStack.get_raw">[docs]</a>
539
+ <span class="k">def</span><span class="w"> </span><span class="nf">get_raw</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">local_resource</span><span class="p">:</span> <span class="n">Resource</span><span class="p">,</span> <span class="n">local_dir</span><span class="p">:</span> <span class="n">Path</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Path</span><span class="p">:</span>
540
+ <span class="w"> </span><span class="sd">&quot;&quot;&quot;</span>
541
+ <span class="sd"> Copies the file stack to a local directory, assumes no context.</span>
542
+
543
+ <span class="sd"> :param local_resource: the local resource to copy the file stack to. Defaults to the current resource.</span>
544
+ <span class="sd"> :param local_dir: The local directory to copy the file stack to.</span>
545
+ <span class="sd"> :type local_dir: Path</span>
546
+ <span class="sd"> &quot;&quot;&quot;</span>
547
+
548
+ <span class="c1"># select the best instance</span>
549
+ <span class="c1"># first search for an instance with &quot;in_memory&quot; set to True</span>
550
+
551
+ <span class="k">if</span> <span class="n">local_dir</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
552
+ <span class="n">local_dir</span> <span class="o">=</span> <span class="n">Path</span><span class="o">.</span><span class="n">cwd</span><span class="p">()</span>
553
+
554
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">in_memory</span><span class="p">:</span>
555
+ <span class="n">local_dir</span><span class="o">.</span><span class="n">mkdir</span><span class="p">(</span><span class="n">parents</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">exist_ok</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
556
+ <span class="k">try</span><span class="p">:</span>
557
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">content</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
558
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;FileStack is in-memory but content is missing.&quot;</span><span class="p">)</span>
559
+ <span class="n">file_name</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">name</span> <span class="ow">or</span> <span class="s2">&quot;file&quot;</span>
560
+ <span class="c1"># Decompress the content</span>
561
+ <span class="n">decompressed_content</span> <span class="o">=</span> <span class="n">zlib</span><span class="o">.</span><span class="n">decompress</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">content</span><span class="p">)</span>
562
+ <span class="c1"># Write the decompressed content to the local directory</span>
563
+ <span class="n">target_path</span> <span class="o">=</span> <span class="n">local_dir</span> <span class="o">/</span> <span class="n">file_name</span>
564
+ <span class="k">if</span> <span class="n">target_path</span><span class="o">.</span><span class="n">exists</span><span class="p">():</span> <span class="c1"># the user must make sure that the file does not exist, otherwise the file will be overwritten</span>
565
+ <span class="k">raise</span> <span class="ne">FileExistsError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;File </span><span class="si">{</span><span class="n">target_path</span><span class="si">}</span><span class="s2"> already exists.&quot;</span><span class="p">)</span>
566
+ <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">target_path</span><span class="p">,</span> <span class="s2">&quot;wb&quot;</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
567
+ <span class="n">f</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">decompressed_content</span><span class="p">)</span>
568
+ <span class="k">return</span> <span class="n">target_path</span>
569
+ <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
570
+ <span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Failed to decompress and write file </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
571
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
572
+ <span class="sa">f</span><span class="s2">&quot;Failed to decompress and write file </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">&quot;</span>
573
+ <span class="p">)</span>
574
+
575
+ <span class="n">local_resource_name</span> <span class="o">=</span> <span class="n">transfer_resource_name</span><span class="p">(</span><span class="n">local_resource</span><span class="p">)</span>
576
+ <span class="n">local_transfer_resource</span> <span class="o">=</span> <span class="n">Resource</span><span class="p">(</span><span class="n">value</span><span class="o">=</span><span class="n">local_resource_name</span><span class="p">)</span>
577
+
578
+ <span class="c1"># If in-memory instance not found or decompression failed, try finding instance on same resource</span>
579
+ <span class="n">same_resource_instance</span> <span class="o">=</span> <span class="nb">next</span><span class="p">(</span>
580
+ <span class="p">(</span>
581
+ <span class="n">f</span>
582
+ <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">locations</span>
583
+ <span class="k">if</span> <span class="n">transfer_resource_name</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">resource</span><span class="p">)</span> <span class="o">==</span> <span class="n">local_resource_name</span>
584
+ <span class="ow">and</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">f</span><span class="p">,</span> <span class="s2">&quot;status&quot;</span><span class="p">,</span> <span class="s2">&quot;available&quot;</span><span class="p">)</span> <span class="o">==</span> <span class="s2">&quot;available&quot;</span>
585
+ <span class="p">),</span>
586
+ <span class="kc">None</span><span class="p">,</span>
587
+ <span class="p">)</span>
588
+
589
+ <span class="k">if</span> <span class="n">same_resource_instance</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
590
+ <span class="n">same_resource_instance</span><span class="o">.</span><span class="n">last_accessed_at</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span>
591
+ <span class="c1"># Return the absolute path by joining with the resource&#39;s workdir if it&#39;s relative</span>
592
+ <span class="n">path</span> <span class="o">=</span> <span class="n">Path</span><span class="p">(</span><span class="n">same_resource_instance</span><span class="o">.</span><span class="n">path</span><span class="p">)</span>
593
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">path</span><span class="o">.</span><span class="n">is_absolute</span><span class="p">():</span>
594
+ <span class="kn">from</span><span class="w"> </span><span class="nn">simstack.core.context</span><span class="w"> </span><span class="kn">import</span> <span class="n">context</span>
595
+
596
+ <span class="k">return</span> <span class="n">Path</span><span class="p">(</span><span class="n">context</span><span class="o">.</span><span class="n">config</span><span class="o">.</span><span class="n">workdir</span><span class="p">)</span> <span class="o">/</span> <span class="n">path</span>
597
+ <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Using existing instance </span><span class="si">{</span><span class="n">path</span><span class="si">}</span><span class="s2"> for </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
598
+ <span class="k">return</span> <span class="n">path</span>
599
+
600
+ <span class="n">remote_instance</span> <span class="o">=</span> <span class="n">first_available_remote_location</span><span class="p">(</span>
601
+ <span class="bp">self</span><span class="o">.</span><span class="n">locations</span><span class="p">,</span> <span class="n">local_transfer_resource</span>
602
+ <span class="p">)</span>
603
+ <span class="k">if</span> <span class="n">remote_instance</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
604
+ <span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;No suitable file instance found for copying.&quot;</span><span class="p">)</span>
605
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
606
+ <span class="sa">f</span><span class="s2">&quot;FileStack </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">id</span><span class="si">}</span><span class="s2"> </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="si">}</span><span class="s2"> is unavailable: no accessible file instance for resource </span><span class="si">{</span><span class="n">local_transfer_resource</span><span class="si">}</span><span class="s2">.&quot;</span>
607
+ <span class="p">)</span>
608
+
609
+ <span class="n">local_dir</span><span class="o">.</span><span class="n">mkdir</span><span class="p">(</span><span class="n">parents</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">exist_ok</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
610
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_via_server_transfer</span><span class="p">(</span>
611
+ <span class="n">remote_instance</span><span class="p">,</span> <span class="n">local_transfer_resource</span><span class="p">,</span> <span class="n">local_dir</span>
612
+ <span class="p">)</span></div>
613
+
614
+
615
+ <span class="k">def</span><span class="w"> </span><span class="nf">_get_via_server_transfer</span><span class="p">(</span>
616
+ <span class="bp">self</span><span class="p">,</span>
617
+ <span class="n">remote_instance</span><span class="p">:</span> <span class="n">FileInstance</span><span class="p">,</span>
618
+ <span class="n">local_resource</span><span class="p">:</span> <span class="n">Resource</span><span class="p">,</span>
619
+ <span class="n">local_dir</span><span class="p">:</span> <span class="n">Path</span><span class="p">,</span>
620
+ <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Path</span><span class="p">:</span>
621
+ <span class="kn">from</span><span class="w"> </span><span class="nn">simstack.core.context</span><span class="w"> </span><span class="kn">import</span> <span class="n">context</span>
622
+
623
+ <span class="n">client</span> <span class="o">=</span> <span class="n">FileTransferClient</span><span class="o">.</span><span class="n">from_context</span><span class="p">(</span><span class="n">required</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
624
+ <span class="k">assert</span> <span class="n">client</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
625
+
626
+ <span class="n">transfer</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">create_transfer</span><span class="p">(</span>
627
+ <span class="n">file_stack_id</span><span class="o">=</span><span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">id</span><span class="p">),</span>
628
+ <span class="n">source_file_instance_id</span><span class="o">=</span><span class="nb">getattr</span><span class="p">(</span><span class="n">remote_instance</span><span class="p">,</span> <span class="s2">&quot;id&quot;</span><span class="p">,</span> <span class="kc">None</span><span class="p">),</span>
629
+ <span class="n">source_resource_name</span><span class="o">=</span><span class="n">transfer_resource_name</span><span class="p">(</span><span class="n">remote_instance</span><span class="o">.</span><span class="n">resource</span><span class="p">),</span>
630
+ <span class="n">target_resource_name</span><span class="o">=</span><span class="n">transfer_resource_name</span><span class="p">(</span><span class="n">local_resource</span><span class="p">),</span>
631
+ <span class="p">)</span>
632
+ <span class="n">transfer_id</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">transfer</span><span class="p">[</span><span class="s2">&quot;transfer_id&quot;</span><span class="p">])</span>
633
+ <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
634
+ <span class="s2">&quot;Created FileStack transfer </span><span class="si">%s</span><span class="s2"> for file_stack=</span><span class="si">%s</span><span class="s2"> from resource=</span><span class="si">%s</span><span class="s2"> to resource=</span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span>
635
+ <span class="n">transfer_id</span><span class="p">,</span>
636
+ <span class="bp">self</span><span class="o">.</span><span class="n">id</span><span class="p">,</span>
637
+ <span class="n">transfer_resource_name</span><span class="p">(</span><span class="n">remote_instance</span><span class="o">.</span><span class="n">resource</span><span class="p">),</span>
638
+ <span class="n">transfer_resource_name</span><span class="p">(</span><span class="n">local_resource</span><span class="p">),</span>
639
+ <span class="p">)</span>
640
+
641
+ <span class="k">try</span><span class="p">:</span>
642
+ <span class="n">client</span><span class="o">.</span><span class="n">wait_until_uploaded</span><span class="p">(</span><span class="n">transfer_id</span><span class="p">)</span>
643
+ <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">exc</span><span class="p">:</span>
644
+ <span class="k">try</span><span class="p">:</span>
645
+ <span class="n">client</span><span class="o">.</span><span class="n">fail_transfer</span><span class="p">(</span>
646
+ <span class="n">transfer_id</span><span class="p">,</span>
647
+ <span class="n">error_message</span><span class="o">=</span><span class="nb">str</span><span class="p">(</span><span class="n">exc</span><span class="p">),</span>
648
+ <span class="n">error_code</span><span class="o">=</span><span class="s2">&quot;SOURCE_RUNNER_UNAVAILABLE&quot;</span><span class="p">,</span>
649
+ <span class="p">)</span>
650
+ <span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span>
651
+ <span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
652
+ <span class="s2">&quot;Failed to report failed FileStack transfer </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">transfer_id</span>
653
+ <span class="p">)</span>
654
+ <span class="k">raise</span>
655
+ <span class="n">target_path</span> <span class="o">=</span> <span class="n">local_dir</span> <span class="o">/</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">name</span> <span class="ow">or</span> <span class="n">Path</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">remote_instance</span><span class="o">.</span><span class="n">path</span><span class="p">))</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
656
+ <span class="k">if</span> <span class="n">target_path</span><span class="o">.</span><span class="n">exists</span><span class="p">():</span>
657
+ <span class="k">raise</span> <span class="ne">FileExistsError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;File </span><span class="si">{</span><span class="n">target_path</span><span class="si">}</span><span class="s2"> already exists.&quot;</span><span class="p">)</span>
658
+ <span class="n">downloaded</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">download_file</span><span class="p">(</span><span class="n">transfer_id</span><span class="p">,</span> <span class="n">target_path</span><span class="p">)</span>
659
+ <span class="n">instance_path</span> <span class="o">=</span> <span class="n">path_for_file_instance</span><span class="p">(</span><span class="n">downloaded</span><span class="o">.</span><span class="n">path</span><span class="p">,</span> <span class="n">context</span><span class="o">.</span><span class="n">config</span><span class="o">.</span><span class="n">workdir</span><span class="p">)</span>
660
+ <span class="n">completed</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">complete_transfer</span><span class="p">(</span>
661
+ <span class="n">transfer_id</span><span class="o">=</span><span class="n">transfer_id</span><span class="p">,</span>
662
+ <span class="n">target_resource_name</span><span class="o">=</span><span class="n">transfer_resource_name</span><span class="p">(</span><span class="n">local_resource</span><span class="p">),</span>
663
+ <span class="n">target_path</span><span class="o">=</span><span class="n">instance_path</span><span class="p">,</span>
664
+ <span class="n">size_bytes</span><span class="o">=</span><span class="n">downloaded</span><span class="o">.</span><span class="n">size_bytes</span><span class="p">,</span>
665
+ <span class="n">checksum_sha256</span><span class="o">=</span><span class="n">downloaded</span><span class="o">.</span><span class="n">checksum_sha256</span><span class="p">,</span>
666
+ <span class="p">)</span>
667
+
668
+ <span class="n">file_instance_id</span> <span class="o">=</span> <span class="n">completed</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;file_instance_id&quot;</span><span class="p">)</span>
669
+ <span class="n">existing</span> <span class="o">=</span> <span class="nb">next</span><span class="p">(</span>
670
+ <span class="p">(</span>
671
+ <span class="n">location</span>
672
+ <span class="k">for</span> <span class="n">location</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">locations</span>
673
+ <span class="k">if</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">location</span><span class="p">,</span> <span class="s2">&quot;id&quot;</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span> <span class="o">==</span> <span class="n">file_instance_id</span>
674
+ <span class="p">),</span>
675
+ <span class="kc">None</span><span class="p">,</span>
676
+ <span class="p">)</span>
677
+ <span class="k">if</span> <span class="n">existing</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
678
+ <span class="bp">self</span><span class="o">.</span><span class="n">locations</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
679
+ <span class="n">FileInstance</span><span class="p">(</span>
680
+ <span class="nb">id</span><span class="o">=</span><span class="nb">str</span><span class="p">(</span><span class="n">file_instance_id</span><span class="p">)</span> <span class="k">if</span> <span class="n">file_instance_id</span> <span class="k">else</span> <span class="kc">None</span><span class="p">,</span>
681
+ <span class="n">path</span><span class="o">=</span><span class="n">instance_path</span><span class="p">,</span>
682
+ <span class="n">resource</span><span class="o">=</span><span class="n">local_resource</span><span class="p">,</span>
683
+ <span class="n">created_at</span><span class="o">=</span><span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">(),</span>
684
+ <span class="n">size_bytes</span><span class="o">=</span><span class="n">downloaded</span><span class="o">.</span><span class="n">size_bytes</span><span class="p">,</span>
685
+ <span class="n">checksum_sha256</span><span class="o">=</span><span class="n">downloaded</span><span class="o">.</span><span class="n">checksum_sha256</span><span class="p">,</span>
686
+ <span class="n">location_type</span><span class="o">=</span><span class="s2">&quot;local_path&quot;</span><span class="p">,</span>
687
+ <span class="n">is_cached</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
688
+ <span class="n">status</span><span class="o">=</span><span class="s2">&quot;available&quot;</span><span class="p">,</span>
689
+ <span class="p">)</span>
690
+ <span class="p">)</span>
691
+
692
+ <span class="k">return</span> <span class="n">downloaded</span><span class="o">.</span><span class="n">path</span>
693
+
694
+ <div class="viewcode-block" id="FileStack.str">
695
+ <a class="viewcode-back" href="../../../simstack/modules/simstack.models.html#simstack.models.files.FileStack.str">[docs]</a>
696
+ <span class="k">def</span><span class="w"> </span><span class="nf">str</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
697
+ <span class="k">return</span> <span class="sa">f</span><span class="s2">&quot;FileStack(name=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="si">}</span><span class="s2">, size=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">size</span><span class="si">}</span><span class="s2">, is_hashable=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">is_hashable</span><span class="si">}</span><span class="s2">, in_memory=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">in_memory</span><span class="si">}</span><span class="s2">, locations=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">locations</span><span class="si">}</span><span class="s2">)&quot;</span></div>
698
+ </div>
699
+
700
+
701
+
702
+ <div class="viewcode-block" id="FileGetterArgs">
703
+ <a class="viewcode-back" href="../../../simstack/modules/simstack.models.html#simstack.models.files.FileGetterArgs">[docs]</a>
704
+ <span class="k">class</span><span class="w"> </span><span class="nc">FileGetterArgs</span><span class="p">(</span><span class="n">Model</span><span class="p">):</span>
705
+ <span class="n">file_stack</span><span class="p">:</span> <span class="n">FileStack</span> <span class="o">=</span> <span class="n">Reference</span><span class="p">()</span>
706
+ <span class="n">local_resource</span><span class="p">:</span> <span class="n">Resource</span>
707
+ <span class="n">local_dir</span><span class="p">:</span> <span class="n">Path</span></div>
708
+
709
+
710
+
711
+ <div class="viewcode-block" id="main">
712
+ <a class="viewcode-back" href="../../../simstack/modules/simstack.models.html#simstack.models.files.main">[docs]</a>
713
+ <span class="k">async</span> <span class="k">def</span><span class="w"> </span><span class="nf">main</span><span class="p">()</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
714
+ <span class="kn">from</span><span class="w"> </span><span class="nn">simstack.core.context</span><span class="w"> </span><span class="kn">import</span> <span class="n">context</span>
715
+
716
+ <span class="k">await</span> <span class="n">context</span><span class="o">.</span><span class="n">initialize</span><span class="p">()</span>
717
+
718
+ <span class="c1"># write a file test.txt</span>
719
+ <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="s2">&quot;test.txt&quot;</span><span class="p">,</span> <span class="s2">&quot;w&quot;</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
720
+ <span class="n">f</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s2">&quot;Hello World&quot;</span><span class="p">)</span>
721
+ <span class="n">file_stack</span> <span class="o">=</span> <span class="n">FileStack</span><span class="o">.</span><span class="n">from_local_file</span><span class="p">(</span><span class="s2">&quot;test.txt&quot;</span><span class="p">,</span> <span class="n">is_hashable</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">in_memory</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
722
+ <span class="nb">print</span><span class="p">(</span><span class="n">file_stack</span><span class="p">)</span>
723
+ <span class="k">await</span> <span class="n">context</span><span class="o">.</span><span class="n">db</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="n">file_stack</span><span class="p">)</span>
724
+ <span class="n">local_dir</span> <span class="o">=</span> <span class="n">Path</span><span class="p">(</span><span class="n">context</span><span class="o">.</span><span class="n">config</span><span class="o">.</span><span class="n">workdir</span><span class="p">)</span> <span class="o">/</span> <span class="s2">&quot;samira&quot;</span> <span class="o">/</span> <span class="nb">str</span><span class="p">(</span><span class="n">file_stack</span><span class="o">.</span><span class="n">id</span><span class="p">)</span>
725
+ <span class="n">retrieved</span> <span class="o">=</span> <span class="n">file_stack</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">local_dir</span><span class="o">=</span><span class="n">local_dir</span><span class="p">)</span>
726
+ <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Retrieved file path:&quot;</span><span class="p">,</span> <span class="n">retrieved</span><span class="p">)</span></div>
727
+
728
+
729
+
730
+ <span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">&quot;__main__&quot;</span><span class="p">:</span>
731
+ <span class="kn">import</span><span class="w"> </span><span class="nn">asyncio</span>
732
+
733
+ <span class="n">logging</span><span class="o">.</span><span class="n">basicConfig</span><span class="p">(</span><span class="n">level</span><span class="o">=</span><span class="n">logging</span><span class="o">.</span><span class="n">DEBUG</span><span class="p">)</span>
734
+ <span class="n">asyncio</span><span class="o">.</span><span class="n">run</span><span class="p">(</span><span class="n">main</span><span class="p">())</span>
735
+ </pre></div>
736
+ </article>
737
+ </div>
738
+ <footer>
739
+
740
+ <div class="related-pages">
741
+
742
+
743
+ </div>
744
+ <div class="bottom-of-page">
745
+ <div class="left-details">
746
+ <div class="copyright">
747
+ Copyright &#169; 2026, Wolfgang Wenzel
748
+ </div>
749
+ Made with <a href="https://www.sphinx-doc.org/">Sphinx</a> and <a class="muted-link" href="https://pradyunsg.me">@pradyunsg</a>'s
750
+
751
+ <a href="https://github.com/pradyunsg/furo">Furo</a>
752
+
753
+ </div>
754
+ <div class="right-details">
755
+
756
+ </div>
757
+ </div>
758
+
759
+ </footer>
760
+ </div>
761
+ <aside class="toc-drawer no-toc">
762
+
763
+
764
+
765
+ </aside>
766
+ </div>
767
+ </div><script src="../../../_static/documentation_options.js?v=5929fcd5"></script>
768
+ <script src="../../../_static/doctools.js?v=fd6eb6e6"></script>
769
+ <script src="../../../_static/sphinx_highlight.js?v=6ffebe34"></script>
770
+ <script src="../../../_static/scripts/furo.js?v=46bd48cc"></script>
771
+ <script data-docs-built-at="2026-07-17 10:50:10 UTC" src="../../../_static/docs-build-badge.js?v=57cb9fc6"></script>
772
+ </body>
773
+ </html>