ob-metaflow-extensions 1.4.17__tar.gz → 1.4.18__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of ob-metaflow-extensions might be problematic. Click here for more details.

Files changed (141) hide show
  1. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/PKG-INFO +1 -1
  2. ob_metaflow_extensions-1.4.18/metaflow_extensions/outerbounds/plugins/s3_proxy/binary_caller.py +132 -0
  3. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/s3_proxy/constants.py +4 -1
  4. ob_metaflow_extensions-1.4.18/metaflow_extensions/outerbounds/plugins/s3_proxy/proxy_bootstrap.py +59 -0
  5. ob_metaflow_extensions-1.4.18/metaflow_extensions/outerbounds/plugins/s3_proxy/s3_proxy_decorator.py +250 -0
  6. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/s3_proxy/s3_proxy_manager.py +44 -41
  7. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.py +32 -4
  8. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/ob_metaflow_extensions.egg-info/PKG-INFO +1 -1
  9. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/ob_metaflow_extensions.egg-info/SOURCES.txt +2 -0
  10. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/setup.py +1 -1
  11. ob_metaflow_extensions-1.4.17/metaflow_extensions/outerbounds/plugins/s3_proxy/s3_proxy_decorator.py +0 -106
  12. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/MANIFEST.in +0 -0
  13. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/README.md +0 -0
  14. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/__init__.py +0 -0
  15. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/config/__init__.py +0 -0
  16. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/__init__.py +0 -0
  17. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/apps/__init__.py +0 -0
  18. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/apps/app_cli.py +0 -0
  19. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/apps/app_deploy_decorator.py +0 -0
  20. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/apps/app_utils.py +0 -0
  21. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/apps/consts.py +0 -0
  22. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/apps/core/__init__.py +0 -0
  23. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/apps/core/_state_machine.py +0 -0
  24. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/apps/core/_vendor/__init__.py +0 -0
  25. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/apps/core/_vendor/spinner/__init__.py +0 -0
  26. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/apps/core/_vendor/spinner/spinners.py +0 -0
  27. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/apps/core/app_cli.py +0 -0
  28. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/apps/core/app_config.py +0 -0
  29. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/apps/core/artifacts.py +0 -0
  30. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/apps/core/capsule.py +0 -0
  31. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/apps/core/click_importer.py +0 -0
  32. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/apps/core/code_package/__init__.py +0 -0
  33. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/apps/core/code_package/code_packager.py +0 -0
  34. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/apps/core/code_package/examples.py +0 -0
  35. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/apps/core/config/__init__.py +0 -0
  36. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/apps/core/config/cli_generator.py +0 -0
  37. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/apps/core/config/config_utils.py +0 -0
  38. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/apps/core/config/schema_export.py +0 -0
  39. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/apps/core/config/typed_configs.py +0 -0
  40. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/apps/core/config/typed_init_generator.py +0 -0
  41. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/apps/core/config/unified_config.py +0 -0
  42. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/apps/core/config_schema.yaml +0 -0
  43. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/apps/core/dependencies.py +0 -0
  44. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/apps/core/deployer.py +0 -0
  45. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/apps/core/experimental/__init__.py +0 -0
  46. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/apps/core/perimeters.py +0 -0
  47. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/apps/core/secrets.py +0 -0
  48. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/apps/core/utils.py +0 -0
  49. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/apps/core/validations.py +0 -0
  50. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/apps/deploy_decorator.py +0 -0
  51. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/apps/supervisord_utils.py +0 -0
  52. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/auth_server.py +0 -0
  53. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/aws/__init__.py +0 -0
  54. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/aws/assume_role.py +0 -0
  55. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/aws/assume_role_decorator.py +0 -0
  56. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/card_utilities/__init__.py +0 -0
  57. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/card_utilities/async_cards.py +0 -0
  58. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/card_utilities/extra_components.py +0 -0
  59. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/card_utilities/injector.py +0 -0
  60. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/checkpoint_datastores/__init__.py +0 -0
  61. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/checkpoint_datastores/coreweave.py +0 -0
  62. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/checkpoint_datastores/external_chckpt.py +0 -0
  63. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/checkpoint_datastores/nebius.py +0 -0
  64. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/fast_bakery/__init__.py +0 -0
  65. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/fast_bakery/baker.py +0 -0
  66. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/fast_bakery/docker_environment.py +0 -0
  67. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/fast_bakery/fast_bakery.py +0 -0
  68. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/fast_bakery/fast_bakery_cli.py +0 -0
  69. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/fast_bakery/fast_bakery_decorator.py +0 -0
  70. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/kubernetes/__init__.py +0 -0
  71. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/kubernetes/kubernetes_client.py +0 -0
  72. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/kubernetes/pod_killer.py +0 -0
  73. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/nim/card.py +0 -0
  74. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/nim/nim_decorator.py +0 -0
  75. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/nim/nim_manager.py +0 -0
  76. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/nim/utils.py +0 -0
  77. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/nvcf/__init__.py +0 -0
  78. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/nvcf/constants.py +0 -0
  79. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/nvcf/exceptions.py +0 -0
  80. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/nvcf/heartbeat_store.py +0 -0
  81. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/nvcf/nvcf.py +0 -0
  82. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/nvcf/nvcf_cli.py +0 -0
  83. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/nvcf/nvcf_decorator.py +0 -0
  84. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/nvcf/utils.py +0 -0
  85. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/nvct/__init__.py +0 -0
  86. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/nvct/exceptions.py +0 -0
  87. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/nvct/nvct.py +0 -0
  88. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/nvct/nvct_cli.py +0 -0
  89. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/nvct/nvct_decorator.py +0 -0
  90. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/nvct/nvct_runner.py +0 -0
  91. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/nvct/utils.py +0 -0
  92. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/ollama/__init__.py +0 -0
  93. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/ollama/constants.py +0 -0
  94. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/ollama/exceptions.py +0 -0
  95. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/ollama/ollama.py +0 -0
  96. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/ollama/status_card.py +0 -0
  97. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/optuna/__init__.py +0 -0
  98. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/perimeters.py +0 -0
  99. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/profilers/deco_injector.py +0 -0
  100. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/profilers/gpu_profile_decorator.py +0 -0
  101. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/profilers/simple_card_decorator.py +0 -0
  102. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/s3_proxy/__init__.py +0 -0
  103. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/s3_proxy/exceptions.py +0 -0
  104. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/s3_proxy/s3_proxy_api.py +0 -0
  105. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/secrets/__init__.py +0 -0
  106. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/secrets/secrets.py +0 -0
  107. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/snowflake/__init__.py +0 -0
  108. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/snowflake/snowflake.py +0 -0
  109. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/snowpark/__init__.py +0 -0
  110. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/snowpark/snowpark.py +0 -0
  111. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/snowpark/snowpark_cli.py +0 -0
  112. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/snowpark/snowpark_client.py +0 -0
  113. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/snowpark/snowpark_decorator.py +0 -0
  114. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/snowpark/snowpark_exceptions.py +0 -0
  115. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/snowpark/snowpark_job.py +0 -0
  116. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/snowpark/snowpark_service_spec.py +0 -0
  117. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/tensorboard/__init__.py +0 -0
  118. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/torchtune/__init__.py +0 -0
  119. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/vllm/__init__.py +0 -0
  120. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/vllm/constants.py +0 -0
  121. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/vllm/exceptions.py +0 -0
  122. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/vllm/status_card.py +0 -0
  123. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/plugins/vllm/vllm_manager.py +0 -0
  124. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/profilers/__init__.py +0 -0
  125. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/profilers/gpu.py +0 -0
  126. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/remote_config.py +0 -0
  127. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/toplevel/__init__.py +0 -0
  128. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/toplevel/ob_internal.py +0 -0
  129. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/toplevel/plugins/azure/__init__.py +0 -0
  130. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/toplevel/plugins/gcp/__init__.py +0 -0
  131. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/toplevel/plugins/kubernetes/__init__.py +0 -0
  132. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/toplevel/plugins/ollama/__init__.py +0 -0
  133. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/toplevel/plugins/optuna/__init__.py +0 -0
  134. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/toplevel/plugins/snowflake/__init__.py +0 -0
  135. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/toplevel/plugins/torchtune/__init__.py +0 -0
  136. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/toplevel/plugins/vllm/__init__.py +0 -0
  137. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/metaflow_extensions/outerbounds/toplevel/s3_proxy.py +0 -0
  138. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/ob_metaflow_extensions.egg-info/dependency_links.txt +0 -0
  139. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/ob_metaflow_extensions.egg-info/requires.txt +0 -0
  140. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/ob_metaflow_extensions.egg-info/top_level.txt +0 -0
  141. {ob_metaflow_extensions-1.4.17 → ob_metaflow_extensions-1.4.18}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: ob_metaflow_extensions
3
- Version: 1.4.17
3
+ Version: 1.4.18
4
4
  Summary: Outerbounds Platform Extensions for Metaflow
5
5
  Author: Outerbounds, Inc.
6
6
  License: Commercial
@@ -0,0 +1,132 @@
1
+ import sys
2
+ import os
3
+ import subprocess
4
+ from metaflow.mflog.mflog import decorate
5
+ from metaflow.mflog import TASK_LOG_SOURCE
6
+ from typing import Union, TextIO, BinaryIO, Callable, Optional
7
+ from queue import Queue, Empty
8
+ from concurrent.futures import ThreadPoolExecutor
9
+ import subprocess
10
+
11
+
12
+ def enqueue_output(file, queue):
13
+ for line in iter(file.readline, ""):
14
+ queue.put(line)
15
+ file.close()
16
+
17
+
18
+ def read_popen_pipes(p: subprocess.Popen):
19
+
20
+ with ThreadPoolExecutor(2) as pool:
21
+ q_stdout, q_stderr = Queue(), Queue()
22
+ pool.submit(enqueue_output, p.stdout, q_stdout)
23
+ pool.submit(enqueue_output, p.stderr, q_stderr)
24
+ while True:
25
+
26
+ if p.poll() is not None and q_stdout.empty() and q_stderr.empty():
27
+ break
28
+
29
+ out_line = err_line = ""
30
+
31
+ try:
32
+ out_line = q_stdout.get_nowait()
33
+ except Empty:
34
+ pass
35
+ try:
36
+ err_line = q_stderr.get_nowait()
37
+ except Empty:
38
+ pass
39
+
40
+ yield (out_line, err_line)
41
+
42
+
43
+ class LogBroadcaster:
44
+ def __init__(
45
+ self,
46
+ process: subprocess.Popen,
47
+ ):
48
+ self._process = process
49
+ self._file_descriptors_and_parsers = []
50
+
51
+ def add_channel(
52
+ self, file_path: str, parser: Optional[Callable[[str], str]] = None
53
+ ):
54
+ self._file_descriptors_and_parsers.append((open(file_path, "a"), parser))
55
+
56
+ def _broadcast_lines(self, out_line: str, err_line: str):
57
+ for file_descriptor, parser in self._file_descriptors_and_parsers:
58
+ if out_line != "":
59
+ if parser:
60
+ out_line = parser(out_line)
61
+ print(out_line, file=file_descriptor, end="", flush=True)
62
+ if err_line != "":
63
+ if parser:
64
+ err_line = parser(err_line)
65
+ print(err_line, file=file_descriptor, end="", flush=True)
66
+
67
+ def publish_line(self, out_line: str, err_line: str):
68
+ self._broadcast_lines(out_line, err_line)
69
+
70
+ def broadcast_logs_to_files(self):
71
+ for out_line, err_line in read_popen_pipes(self._process):
72
+ self._broadcast_lines(out_line, err_line)
73
+
74
+ self._process.wait()
75
+
76
+ for file_descriptor, _ in self._file_descriptors_and_parsers:
77
+ file_descriptor.close()
78
+
79
+
80
+ def run_with_mflog_capture(command, debug=False):
81
+ """
82
+ Run a subprocess with proper mflog integration for stdout/stderr capture.
83
+ This mimics what bash_capture_logs does but in Python.
84
+ """
85
+ # Get the log file paths from environment variables
86
+ stdout_path = os.environ.get("MFLOG_STDOUT")
87
+ stderr_path = os.environ.get("MFLOG_STDERR")
88
+
89
+ if not stdout_path or not stderr_path:
90
+ # Fallback to regular subprocess if mflog env vars aren't set
91
+ return subprocess.run(command, check=True, shell=True)
92
+
93
+ pipe = subprocess.PIPE if debug else subprocess.DEVNULL
94
+ # Start the subprocess with pipes
95
+ process = subprocess.Popen(
96
+ command,
97
+ shell=True,
98
+ stdout=pipe,
99
+ stderr=pipe,
100
+ text=False, # Use bytes for proper mflog handling
101
+ bufsize=0, # Unbuffered for real-time logging
102
+ )
103
+
104
+ broadcaster = LogBroadcaster(process)
105
+
106
+ broadcaster.add_channel(
107
+ stderr_path, lambda line: decorate(TASK_LOG_SOURCE, line).decode("utf-8")
108
+ )
109
+ broadcaster.publish_line(f"[S3 PROXY] Starting Fast S3 Proxy.....\n", "")
110
+ broadcaster.broadcast_logs_to_files()
111
+
112
+ # Check the return code and raise if non-zero
113
+ if process.returncode != 0:
114
+ raise subprocess.CalledProcessError(process.returncode, command)
115
+
116
+ return process.returncode
117
+
118
+
119
+ if __name__ == "__main__":
120
+ s3_proxy_binary_path = os.environ.get("S3_PROXY_BINARY_COMMAND")
121
+ s3_proxy_debug = bool(os.environ.get("S3_PROXY_BINARY_DEBUG", False))
122
+ if not s3_proxy_binary_path:
123
+ print("S3_PROXY_BINARY_COMMAND environment variable not set")
124
+ sys.exit(1)
125
+
126
+ try:
127
+ run_with_mflog_capture(s3_proxy_binary_path, debug=s3_proxy_debug)
128
+ except subprocess.CalledProcessError as e:
129
+ sys.exit(e.returncode)
130
+ except Exception as e:
131
+ print(f"Error running S3 proxy binary: {e}", file=sys.stderr)
132
+ sys.exit(1)
@@ -5,4 +5,7 @@ S3_PROXY_BINARY_URLS = {
5
5
 
6
6
  DEFAULT_PROXY_PORT = 8081
7
7
  DEFAULT_PROXY_HOST = "localhost"
8
- S3_PROXY_WRITE_MODES = ["origin-and-cache", "origin", "cache"]
8
+ S3_PROXY_WRITE_MODES = [
9
+ "origin-and-cache",
10
+ "origin",
11
+ ]
@@ -0,0 +1,59 @@
1
+ from .s3_proxy_manager import S3ProxyManager
2
+ from metaflow._vendor import click
3
+ from metaflow import JSONType
4
+ import json
5
+
6
+
7
+ @click.group()
8
+ def cli():
9
+ pass
10
+
11
+
12
+ @cli.command()
13
+ @click.option(
14
+ "--integration-name", type=str, help="The integration name", required=True
15
+ )
16
+ @click.option("--write-mode", type=str, help="The write mode")
17
+ @click.option("--debug", type=bool, help="The debug mode", default=False)
18
+ @click.option(
19
+ "--uc-proxy-cfg-write-path",
20
+ type=str,
21
+ help="The path to write the user code proxy config",
22
+ required=True,
23
+ )
24
+ @click.option(
25
+ "--proxy-status-write-path",
26
+ type=str,
27
+ help="The path to write the proxy status",
28
+ required=True,
29
+ )
30
+ def bootstrap(
31
+ integration_name,
32
+ write_mode,
33
+ debug,
34
+ uc_proxy_cfg_write_path,
35
+ proxy_status_write_path,
36
+ ):
37
+ manager = S3ProxyManager(
38
+ integration_name=integration_name,
39
+ write_mode=write_mode,
40
+ debug=debug,
41
+ )
42
+ user_code_proxy_config, proxy_pid, config_path, binary_path = manager.setup_proxy()
43
+ with open(uc_proxy_cfg_write_path, "w") as f:
44
+ f.write(json.dumps(user_code_proxy_config))
45
+ with open(proxy_status_write_path, "w") as f:
46
+ f.write(
47
+ json.dumps(
48
+ {
49
+ "proxy_pid": proxy_pid,
50
+ "config_path": config_path,
51
+ "binary_path": binary_path,
52
+ }
53
+ )
54
+ )
55
+
56
+
57
+ if __name__ == "__main__":
58
+ print("[@s3_proxy] Jumpstarting the proxy....")
59
+ cli()
@@ -0,0 +1,250 @@
1
+ import os
2
+ import functools
3
+ import json
4
+ import signal
5
+ import time
6
+ from typing import Dict, List, Optional, Tuple
7
+
8
+ from metaflow import current
9
+ from metaflow.decorators import StepDecorator
10
+ from .exceptions import S3ProxyException
11
+ from .constants import S3_PROXY_WRITE_MODES
12
+ from collections import namedtuple
13
+
14
+ S3ProxyBinaryConfig = namedtuple(
15
+ "S3ProxyBinaryConfig", ["integration_name", "write_mode", "debug"]
16
+ )
17
+
18
+
19
+ def monkey_patch_environment(
20
+ environment, step_name_and_deco_attrs: Dict[str, S3ProxyBinaryConfig]
21
+ ):
22
+ wrapping_func = environment.bootstrap_commands
23
+
24
+ @functools.wraps(wrapping_func)
25
+ def wrapper(step_name, ds_type, *args):
26
+ base_boostrap_cmd = wrapping_func(step_name, ds_type, *args)
27
+ additional_cmd = []
28
+
29
+ if step_name in step_name_and_deco_attrs:
30
+ integration_name = step_name_and_deco_attrs[step_name].integration_name
31
+ write_mode = step_name_and_deco_attrs[step_name].write_mode
32
+ debug = step_name_and_deco_attrs[step_name].debug
33
+ additional_cmd = [
34
+ "echo 'Setting up the S3 proxy.'",
35
+ f"python -m metaflow_extensions.outerbounds.plugins.s3_proxy.proxy_bootstrap bootstrap --integration-name {integration_name} --write-mode {write_mode} --debug {debug} --uc-proxy-cfg-write-path ./.uc_proxy_cfg_file --proxy-status-write-path ./.proxy_status_file",
36
+ "export METAFLOW_S3_PROXY_USER_CODE_CONFIG=$(cat ./.uc_proxy_cfg_file)",
37
+ "export METAFLOW_S3_PROXY_STATUS=$(cat ./.proxy_status_file)",
38
+ "export METAFLOW_S3_PROXY_SETUP_SUCCESS=True",
39
+ "flush_mflogs",
40
+ ]
41
+ return base_boostrap_cmd + additional_cmd
42
+
43
+ environment.bootstrap_commands = wrapper
44
+
45
+
46
+ class S3ProxyDecorator(StepDecorator):
47
+ """
48
+ Set up an S3 proxy that caches objects in an external, S3‑compatible bucket
49
+ for S3 read and write requests.
50
+
51
+ This decorator requires an integration in the Outerbounds platform that
52
+ points to an external bucket. It affects S3 operations performed via
53
+ Metaflow's `get_aws_client` and `S3` within a `@step`.
54
+
55
+ Read operations
56
+ ---------------
57
+ All read operations pass through the proxy. If an object does not already
58
+ exist in the external bucket, it is cached there. For example, if code reads
59
+ from buckets `FOO` and `BAR` using the `S3` interface, objects from both
60
+ buckets are cached in the external bucket.
61
+
62
+ During task execution, all S3‑related read requests are routed through the
63
+ proxy:
64
+ - If the object is present in the external object store, the proxy
65
+ streams it directly from there without accessing the requested origin
66
+ bucket.
67
+ - If the object is not present in the external storage, the proxy
68
+ fetches it from the requested bucket, caches it in the external
69
+ storage, and streams the response from the origin bucket.
70
+
71
+ Warning
72
+ -------
73
+ All READ operations (e.g., GetObject, HeadObject) pass through the external
74
+ bucket regardless of the bucket specified in user code. Even
75
+ `S3(run=self)` and `S3(s3root="mybucketfoo")` requests go through the
76
+ external bucket cache.
77
+
78
+ Write operations
79
+ ----------------
80
+ Write behavior is controlled by the `write_mode` parameter, which determines
81
+ whether writes also persist objects in the cache.
82
+
83
+ `write_mode` values:
84
+ - `origin-and-cache`: objects are written both to the cache and to their
85
+ intended origin bucket.
86
+ - `origin`: objects are written only to their intended origin bucket.
87
+
88
+ Parameters
89
+ ----------
90
+ integration_name : str, optional
91
+ [Outerbounds integration name](https://docs.outerbounds.com/outerbounds/configuring-secrets/#integrations-view)
92
+ that holds the configuration for the external, S3‑compatible object
93
+ storage bucket. If not specified, the only available S3 proxy
94
+ integration in the namespace is used (fails if multiple exist).
95
+ write_mode : str, optional
96
+ Controls whether writes also go to the external bucket.
97
+ - `origin` (default)
98
+ - `origin-and-cache`
99
+ debug : bool, optional
100
+ Enables debug logging for proxy operations.
101
+ """
102
+
103
+ name = "s3_proxy"
104
+ defaults = {
105
+ "integration_name": None,
106
+ "write_mode": None,
107
+ "debug": False,
108
+ }
109
+
110
+ _environment_patched = False
111
+
112
+ _proxy_status = None
113
+
114
+ @classmethod
115
+ def patch_environment(cls, flow, environment):
116
+ """
117
+ We need to patch the environment boostrap command so that
118
+ we can launch the s3 proxy before the step code execution.
119
+ We also want to ensure that we are running the proxy bootstrap
120
+ only for the steps that have the decorator set. This is why we pass down all
121
+ the step names that will change the boostrap commands.
122
+ """
123
+ if cls._environment_patched:
124
+ return
125
+
126
+ steps_with_s3_proxy = [
127
+ step
128
+ for step in flow
129
+ if any(deco.name == "s3_proxy" for deco in step.decorators)
130
+ ]
131
+ if len(steps_with_s3_proxy) == 0: # weird but y not?
132
+ return
133
+
134
+ step_names_and_deco_attrs = {}
135
+ for s in steps_with_s3_proxy:
136
+ _decos = [x for x in s.decorators if x.name == "s3_proxy"]
137
+ deco = _decos[0]
138
+ step_names_and_deco_attrs[s.name] = S3ProxyBinaryConfig(
139
+ integration_name=deco.attributes["integration_name"],
140
+ write_mode=deco.attributes["write_mode"],
141
+ debug=deco.attributes["debug"],
142
+ )
143
+
144
+ monkey_patch_environment(environment, step_names_and_deco_attrs)
145
+ cls._environment_patched = True
146
+
147
+ def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger):
148
+ write_mode = self.attributes["write_mode"]
149
+ if write_mode and write_mode not in S3_PROXY_WRITE_MODES:
150
+ raise S3ProxyException(
151
+ f"unexpected write_mode specified: {write_mode}. Allowed values are: {','.join(S3_PROXY_WRITE_MODES)}."
152
+ )
153
+
154
+ self.patch_environment(flow, environment)
155
+ if (
156
+ os.environ.get("METAFLOW_S3_PROXY_USER_CODE_CONFIG")
157
+ and os.environ.get("METAFLOW_S3_PROXY_STATUS")
158
+ and self.attributes["debug"]
159
+ ):
160
+ print("[@s3_proxy] S3 Proxy detected. Debug mode is enabled.")
161
+
162
+ if os.environ.get("METAFLOW_S3_PROXY_STATUS"):
163
+ proxy_status = json.loads(os.environ.get("METAFLOW_S3_PROXY_STATUS"))
164
+ self._proxy_status = proxy_status
165
+
166
+ def task_pre_step(
167
+ self,
168
+ step_name,
169
+ task_datastore,
170
+ metadata,
171
+ run_id,
172
+ task_id,
173
+ flow,
174
+ graph,
175
+ retry_count,
176
+ max_user_code_retries,
177
+ ubf_context,
178
+ inputs,
179
+ ):
180
+ """Setup S3 proxy before step execution"""
181
+ pass
182
+
183
+ def task_finished(
184
+ self, step_name, flow, graph, is_task_ok, retry_count, max_retries
185
+ ):
186
+ if not self._proxy_status:
187
+ return
188
+
189
+ status = self._proxy_status
190
+ proxy_pid = status.get("proxy_pid")
191
+ config_path = status.get("config_path")
192
+ binary_path = status.get("binary_path")
193
+
194
+ # 1) Stop processes: try to terminate the process group for clean child shutdown
195
+ if proxy_pid:
196
+ try:
197
+ pgid = os.getpgid(proxy_pid)
198
+ os.killpg(pgid, signal.SIGTERM)
199
+ time.sleep(1)
200
+ except Exception:
201
+ # Fall back to killing the pid directly if pgid is unavailable
202
+ try:
203
+ os.kill(proxy_pid, signal.SIGTERM)
204
+ except Exception:
205
+ pass
206
+
207
+ # 2) Clear files based on status
208
+ for path in (config_path, binary_path):
209
+ try:
210
+ if path and os.path.exists(path):
211
+ os.remove(path)
212
+ except Exception:
213
+ pass
214
+
215
+
216
+ class NebiusS3ProxyDecorator(S3ProxyDecorator):
217
+
218
+ __doc__ = (
219
+ """
220
+ `@nebius_s3_proxy` is a Nebius-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
221
+ It exists to make it easier for users to know that this decorator should only be used with
222
+ a Neo Cloud like Nebius. The underlying mechanics of the decorator is the same as the `@s3_proxy`:\n
223
+ """
224
+ + S3ProxyDecorator.__doc__
225
+ )
226
+
227
+ name = "nebius_s3_proxy"
228
+ defaults = {
229
+ "integration_name": None,
230
+ "write_mode": None,
231
+ "debug": False,
232
+ }
233
+
234
+
235
+ class CoreWeaveS3ProxyDecorator(S3ProxyDecorator):
236
+ __doc__ = (
237
+ """
238
+ `@coreweave_s3_proxy` is a CoreWeave-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
239
+ It exists to make it easier for users to know that this decorator should only be used with
240
+ a Neo Cloud like CoreWeave. The underlying mechanics of the decorator is the same as the `@s3_proxy`:\n
241
+ """
242
+ + S3ProxyDecorator.__doc__
243
+ )
244
+
245
+ name = "coreweave_s3_proxy"
246
+ defaults = {
247
+ "integration_name": None,
248
+ "write_mode": None,
249
+ "debug": False,
250
+ }
@@ -1,11 +1,12 @@
1
1
  import os
2
2
  import json
3
3
  import gzip
4
+ import sys
4
5
  import time
5
6
  import threading
6
7
  import subprocess
7
8
  from pathlib import Path
8
- from typing import Optional
9
+ from typing import Optional, Tuple
9
10
 
10
11
  import requests
11
12
 
@@ -35,7 +36,7 @@ class S3ProxyManager:
35
36
  self.api_client = S3ProxyApiClient()
36
37
  self.proxy_config = None
37
38
 
38
- def setup_proxy(self) -> bool:
39
+ def setup_proxy(self) -> Tuple[dict, int, str, str]:
39
40
  try:
40
41
  if self._is_running_in_kubernetes():
41
42
  config_data = self.api_client.fetch_s3_proxy_config(
@@ -43,14 +44,32 @@ class S3ProxyManager:
43
44
  )
44
45
  self.binary_path = self._download_binary()
45
46
  self.config_path = self._write_config_file(config_data)
47
+ # In the new world where the binary is being called
48
+ # before even the metaflow code exection starts,
49
+ # so this implies a few important things:
50
+ # 1, We start the actual proxy process via another python file that safely ships logs to mflog.
51
+ # 2. We passback the right values to the metaflow step process via env vars.
52
+ # 3. Metaflow step code relies on env vars to decide if clients need to have s3 proxy in them.
46
53
  self.process = self._start_proxy_process()
47
- self._setup_proxy_config(config_data)
48
- return True
54
+
55
+ user_code_proxy_config = self._setup_proxy_config(config_data)
56
+
57
+ return_tuple = (
58
+ user_code_proxy_config, # this is the config that will be used within the metaflow `step` code.
59
+ self.process.pid, # This is the pid of the process that will jumpstart, monitor and ship logs to MFLOG for the proxy process
60
+ self.config_path, # This is the path to the config that is derived from the integration. It contains the actual bucket path and name where external objects are stored.
61
+ self.binary_path, # This is the path to the binary for the proxy.
62
+ )
63
+ # We return a tuple because these values need to be passed down to the metaflow step process where
64
+ # it will handle thier removal gracefully after the step is finished.
65
+ return return_tuple
49
66
 
50
67
  print(
51
68
  "[@s3_proxy] skipping s3-proxy set up because metaflow has not detected a Kubernetes environment"
52
69
  )
53
- return False
70
+ raise S3ProxyException(
71
+ "S3 proxy setup failed because metaflow has not detected a Kubernetes environment"
72
+ )
54
73
  except Exception as e:
55
74
  if self.debug:
56
75
  print(f"[@s3_proxy] Setup failed: {e}")
@@ -121,21 +140,31 @@ class S3ProxyManager:
121
140
  return str(config_path.absolute())
122
141
 
123
142
  def _start_proxy_process(self) -> subprocess.Popen:
143
+ # This command will jump start a process that will then call the proxy binary
144
+ # The reason we do something like this is because we need to run all of this before
145
+ # even the `step` command is called. So we need a python process that will ship the logs
146
+ # of the proxy process to MFLOG instead of setting print statements. We need this process
147
+ # to run independently since the S3ProxyManager gets called in the boostrap_proxy which will
148
+ # exit after jump starting the proxy process.
124
149
  cmd = [self.binary_path, "--bucket-config", self.config_path, "serve"]
125
-
150
+ _env = os.environ.copy()
151
+ _env["S3_PROXY_BINARY_COMMAND"] = " ".join(cmd)
126
152
  if self.debug:
127
- print(f"[@s3_proxy] Starting proxy: {' '.join(cmd)}")
128
-
153
+ _env["S3_PROXY_BINARY_DEBUG"] = "True"
154
+ _cmd = [
155
+ sys.executable,
156
+ "-m",
157
+ "metaflow_extensions.outerbounds.plugins.s3_proxy.binary_caller",
158
+ ]
159
+ devnull = subprocess.DEVNULL
129
160
  process = subprocess.Popen(
130
- cmd,
131
- stdout=subprocess.PIPE,
132
- stderr=subprocess.STDOUT, # Redirect stderr to stdout
161
+ _cmd,
162
+ stdout=devnull,
163
+ stderr=devnull,
133
164
  text=True,
134
165
  start_new_session=True,
166
+ env=_env,
135
167
  )
136
-
137
- self._setup_log_streaming(process)
138
-
139
168
  time.sleep(3)
140
169
 
141
170
  if process.poll() is None:
@@ -149,30 +178,7 @@ class S3ProxyManager:
149
178
  print(f"[@s3_proxy] Proxy failed to start - output: {stdout_data}")
150
179
  raise S3ProxyException(f"S3 proxy failed to start: {stdout_data}")
151
180
 
152
- def _setup_log_streaming(self, process: subprocess.Popen):
153
- def stream_logs():
154
- try:
155
- # Read stdout line by line (stderr is redirected to stdout)
156
- while True:
157
- line = process.stdout.readline()
158
- if not line:
159
- # Process has ended
160
- break
161
- line = line.strip()
162
- if line and self.debug:
163
- print(f"[@s3_proxy] {line}")
164
-
165
- except Exception as e:
166
- if self.debug:
167
- print(f"[@s3_proxy] Log streaming error: {e}")
168
-
169
- log_thread = threading.Thread(target=stream_logs, daemon=True)
170
- log_thread.start()
171
-
172
181
  def _setup_proxy_config(self, config_data):
173
- from metaflow_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package import (
174
- set_s3_proxy_config,
175
- )
176
182
  from metaflow.metaflow_config import AWS_SECRETS_MANAGER_DEFAULT_REGION
177
183
 
178
184
  region = os.environ.get(
@@ -190,11 +196,8 @@ class S3ProxyManager:
190
196
  if self.write_mode:
191
197
  proxy_config["write_mode"] = self.write_mode
192
198
 
193
- set_s3_proxy_config(proxy_config)
194
199
  self.proxy_config = proxy_config
195
-
196
- if self.debug:
197
- print("[@s3_proxy] Global S3 proxy configuration activated")
200
+ return proxy_config
198
201
 
199
202
  def cleanup(self):
200
203
  try:
@@ -24,9 +24,26 @@ def clear_s3_proxy_config():
24
24
 
25
25
 
26
26
  def get_s3_proxy_config():
27
+ global _S3_PROXY_CONFIG
28
+ if _S3_PROXY_CONFIG is None:
29
+ set_s3_proxy_config(get_s3_proxy_config_from_env())
27
30
  return _S3_PROXY_CONFIG
28
31
 
29
32
 
33
+ # TODO: Refactor out the _S3_PROXY_CONFIG global variable and instead use the function that
34
+ # extracts it from the environment variables.
35
+
36
+ import os
37
+ import json
38
+
39
+
40
+ def get_s3_proxy_config_from_env():
41
+ env_conf = os.environ.get("METAFLOW_S3_PROXY_USER_CODE_CONFIG")
42
+ if env_conf:
43
+ return json.loads(env_conf)
44
+ return None
45
+
46
+
30
47
  # Must match the signature of metaflow.plugins.aws.aws_client.get_aws_client
31
48
  # This function is called by the "userland" code inside tasks. Metaflow internals
32
49
  # will call the function in metaflow.plugins.aws.aws_client.get_aws_client directly.
@@ -55,9 +72,14 @@ def get_aws_client(
55
72
  if decorator_role_arn:
56
73
  role_arn = decorator_role_arn
57
74
 
58
- if module == "s3" and _S3_PROXY_CONFIG is not None:
75
+ if module == "s3" and get_s3_proxy_config() is not None:
59
76
  return get_aws_client_with_s3_proxy(
60
- module, with_error, role_arn, session_vars, client_params, _S3_PROXY_CONFIG
77
+ module,
78
+ with_error,
79
+ role_arn,
80
+ session_vars,
81
+ client_params,
82
+ get_s3_proxy_config(),
61
83
  )
62
84
 
63
85
  client = metaflow.plugins.aws.aws_client.get_aws_client(
@@ -97,12 +119,18 @@ def S3(*args, **kwargs):
97
119
  kwargs["role"] = USE_CSPR_ROLE_ARN_IF_SET
98
120
 
99
121
  # Check if S3 proxy is active using module variable (like CSPR)
100
- if _S3_PROXY_CONFIG is not None:
101
- return get_S3_with_s3_proxy(_S3_PROXY_CONFIG, *args, **kwargs)
122
+ if get_s3_proxy_config() is not None:
123
+ return get_S3_with_s3_proxy(get_s3_proxy_config(), *args, **kwargs)
102
124
 
103
125
  return metaflow.plugins.datatools.s3.S3(*args, **kwargs)
104
126
 
105
127
 
128
+ # Setting the S3 client docstring in order to ensure that
129
+ # stubs get generated properly.
130
+ import metaflow.plugins.datatools.s3
131
+
132
+ S3.__doc__ = metaflow.plugins.datatools.s3.S3.__doc__
133
+
106
134
  from .. import profilers
107
135
  from ..plugins.snowflake import Snowflake
108
136
  from ..plugins.checkpoint_datastores import nebius_checkpoints, coreweave_checkpoints
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: ob-metaflow-extensions
3
- Version: 1.4.17
3
+ Version: 1.4.18
4
4
  Summary: Outerbounds Platform Extensions for Metaflow
5
5
  Author: Outerbounds, Inc.
6
6
  License: Commercial
@@ -91,8 +91,10 @@ metaflow_extensions/outerbounds/plugins/profilers/deco_injector.py
91
91
  metaflow_extensions/outerbounds/plugins/profilers/gpu_profile_decorator.py
92
92
  metaflow_extensions/outerbounds/plugins/profilers/simple_card_decorator.py
93
93
  metaflow_extensions/outerbounds/plugins/s3_proxy/__init__.py
94
+ metaflow_extensions/outerbounds/plugins/s3_proxy/binary_caller.py
94
95
  metaflow_extensions/outerbounds/plugins/s3_proxy/constants.py
95
96
  metaflow_extensions/outerbounds/plugins/s3_proxy/exceptions.py
97
+ metaflow_extensions/outerbounds/plugins/s3_proxy/proxy_bootstrap.py
96
98
  metaflow_extensions/outerbounds/plugins/s3_proxy/s3_proxy_api.py
97
99
  metaflow_extensions/outerbounds/plugins/s3_proxy/s3_proxy_decorator.py
98
100
  metaflow_extensions/outerbounds/plugins/s3_proxy/s3_proxy_manager.py