lightning-sdk 0.1.2__py3-none-any.whl → 0.1.46__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (521) hide show
  1. lightning_sdk/__init__.py +12 -2
  2. lightning_sdk/agents.py +46 -0
  3. lightning_sdk/ai_hub.py +185 -0
  4. lightning_sdk/api/__init__.py +4 -0
  5. lightning_sdk/api/agents_api.py +107 -0
  6. lightning_sdk/api/ai_hub_api.py +130 -0
  7. lightning_sdk/api/deployment_api.py +574 -0
  8. lightning_sdk/api/job_api.py +308 -0
  9. lightning_sdk/api/mmt_api.py +188 -0
  10. lightning_sdk/api/org_api.py +1 -3
  11. lightning_sdk/api/studio_api.py +174 -83
  12. lightning_sdk/api/teamspace_api.py +219 -20
  13. lightning_sdk/api/user_api.py +24 -9
  14. lightning_sdk/api/utils.py +429 -47
  15. lightning_sdk/cli/ai_hub.py +49 -0
  16. lightning_sdk/cli/download.py +132 -0
  17. lightning_sdk/cli/entrypoint.py +11 -3
  18. lightning_sdk/cli/run.py +206 -0
  19. lightning_sdk/cli/serve.py +218 -0
  20. lightning_sdk/cli/studios_menu.py +78 -0
  21. lightning_sdk/cli/upload.py +79 -89
  22. lightning_sdk/constants.py +29 -1
  23. lightning_sdk/deployment/__init__.py +25 -0
  24. lightning_sdk/deployment/deployment.py +389 -0
  25. lightning_sdk/helpers.py +49 -0
  26. lightning_sdk/job/__init__.py +5 -0
  27. lightning_sdk/job/base.py +295 -0
  28. lightning_sdk/job/job.py +266 -0
  29. lightning_sdk/job/v1.py +241 -0
  30. lightning_sdk/job/v2.py +193 -0
  31. lightning_sdk/job/work.py +72 -0
  32. lightning_sdk/lightning_cloud/__version__.py +1 -1
  33. lightning_sdk/lightning_cloud/cli/__main__.py +15 -13
  34. lightning_sdk/lightning_cloud/env.py +1 -0
  35. lightning_sdk/lightning_cloud/login.py +12 -8
  36. lightning_sdk/lightning_cloud/openapi/__init__.py +314 -43
  37. lightning_sdk/lightning_cloud/openapi/api/__init__.py +10 -0
  38. lightning_sdk/lightning_cloud/openapi/api/analytics_service_api.py +141 -0
  39. lightning_sdk/lightning_cloud/openapi/api/assistants_service_api.py +1075 -227
  40. lightning_sdk/lightning_cloud/openapi/api/billing_service_api.py +9 -1
  41. lightning_sdk/lightning_cloud/openapi/api/cloud_space_service_api.py +3115 -895
  42. lightning_sdk/lightning_cloud/openapi/api/cluster_service_api.py +908 -141
  43. lightning_sdk/lightning_cloud/openapi/api/data_connection_service_api.py +10 -5
  44. lightning_sdk/lightning_cloud/openapi/api/deployment_templates_service_api.py +756 -0
  45. lightning_sdk/lightning_cloud/openapi/api/endpoint_service_api.py +1658 -159
  46. lightning_sdk/lightning_cloud/openapi/api/experiments_service_api.py +242 -0
  47. lightning_sdk/lightning_cloud/openapi/api/file_system_service_api.py +674 -0
  48. lightning_sdk/lightning_cloud/openapi/api/jobs_service_api.py +3823 -0
  49. lightning_sdk/lightning_cloud/openapi/api/lightningapp_instance_service_api.py +158 -590
  50. lightning_sdk/lightning_cloud/openapi/api/lightningapp_v2_service_api.py +0 -1086
  51. lightning_sdk/lightning_cloud/openapi/api/lightningwork_service_api.py +113 -0
  52. lightning_sdk/lightning_cloud/openapi/api/lit_logger_service_api.py +1753 -0
  53. lightning_sdk/lightning_cloud/openapi/api/lit_registry_service_api.py +242 -0
  54. lightning_sdk/lightning_cloud/openapi/api/models_store_api.py +1423 -108
  55. lightning_sdk/lightning_cloud/openapi/api/organizations_service_api.py +421 -1
  56. lightning_sdk/lightning_cloud/openapi/api/profiler_service_api.py +663 -0
  57. lightning_sdk/lightning_cloud/openapi/api/projects_service_api.py +5 -1
  58. lightning_sdk/lightning_cloud/openapi/api/secret_service_api.py +478 -1
  59. lightning_sdk/lightning_cloud/openapi/api/slurm_jobs_user_service_api.py +9 -5
  60. lightning_sdk/lightning_cloud/openapi/api/snowflake_service_api.py +686 -0
  61. lightning_sdk/lightning_cloud/openapi/api/storage_service_api.py +1094 -0
  62. lightning_sdk/lightning_cloud/openapi/api/studio_jobs_service_api.py +4 -4
  63. lightning_sdk/lightning_cloud/openapi/api/user_service_api.py +1081 -34
  64. lightning_sdk/lightning_cloud/openapi/models/__init__.py +304 -41
  65. lightning_sdk/lightning_cloud/openapi/models/affiliatelinks_id_body.py +149 -0
  66. lightning_sdk/lightning_cloud/openapi/models/agentmanagedendpoints_id_body.py +305 -0
  67. lightning_sdk/lightning_cloud/openapi/models/agents_id_body.py +643 -0
  68. lightning_sdk/lightning_cloud/openapi/models/app_id_works_body.py +149 -0
  69. lightning_sdk/lightning_cloud/openapi/models/approveautojoindomain_domain_body.py +123 -0
  70. lightning_sdk/lightning_cloud/openapi/models/apps_id_body1.py +107 -3
  71. lightning_sdk/lightning_cloud/openapi/models/assistant_id_conversations_body.py +1 -27
  72. lightning_sdk/lightning_cloud/openapi/models/{v1_get_cluster_health_response.py → captures_id_body.py} +16 -16
  73. lightning_sdk/lightning_cloud/openapi/models/cloud_space_id_versionpublications_body.py +175 -0
  74. lightning_sdk/lightning_cloud/openapi/models/cloud_space_id_versionpublications_body1.py +175 -0
  75. lightning_sdk/lightning_cloud/openapi/models/cloud_space_id_versions_body.py +409 -0
  76. lightning_sdk/lightning_cloud/openapi/models/cloudspace_id_runs_body.py +27 -1
  77. lightning_sdk/lightning_cloud/openapi/models/cloudspaces_id_body.py +84 -6
  78. lightning_sdk/lightning_cloud/openapi/models/cluster_id_capacityblock_body.py +253 -0
  79. lightning_sdk/lightning_cloud/openapi/models/cluster_id_capacityreservations_body.py +55 -3
  80. lightning_sdk/lightning_cloud/openapi/models/cluster_id_proxies_body.py +123 -0
  81. lightning_sdk/lightning_cloud/openapi/models/clusters_id_body.py +27 -1
  82. lightning_sdk/lightning_cloud/openapi/models/create.py +157 -1
  83. lightning_sdk/lightning_cloud/openapi/models/create_deployment_request_defines_a_spec_for_the_job_that_allows_for_autoscaling_jobs.py +383 -0
  84. lightning_sdk/lightning_cloud/openapi/models/deployments_id_body.py +565 -0
  85. lightning_sdk/lightning_cloud/openapi/models/deploymenttemplates_id_body.py +513 -0
  86. lightning_sdk/lightning_cloud/openapi/models/endpoints_id_body.py +43 -17
  87. lightning_sdk/lightning_cloud/openapi/models/experiment_name_variant_name_body.py +123 -0
  88. lightning_sdk/lightning_cloud/openapi/models/externalv1_cloud_space_instance_status.py +107 -1
  89. lightning_sdk/lightning_cloud/openapi/models/externalv1_lightningapp_instance.py +27 -1
  90. lightning_sdk/lightning_cloud/openapi/models/externalv1_user_status.py +27 -1
  91. lightning_sdk/lightning_cloud/openapi/models/fileendpoints_id_body.py +131 -1
  92. lightning_sdk/lightning_cloud/openapi/models/id_codeconfig_body.py +29 -55
  93. lightning_sdk/lightning_cloud/openapi/models/id_complete_body.py +175 -0
  94. lightning_sdk/lightning_cloud/openapi/models/id_engage_body.py +3 -29
  95. lightning_sdk/lightning_cloud/openapi/models/id_engage_body1.py +149 -0
  96. lightning_sdk/lightning_cloud/openapi/models/id_execute_body.py +3 -55
  97. lightning_sdk/lightning_cloud/openapi/models/id_execute_body1.py +175 -0
  98. lightning_sdk/lightning_cloud/openapi/models/id_fork_body.py +31 -31
  99. lightning_sdk/lightning_cloud/openapi/models/id_fork_body1.py +175 -0
  100. lightning_sdk/lightning_cloud/openapi/models/id_get_body.py +133 -3
  101. lightning_sdk/lightning_cloud/openapi/models/id_index_body.py +67 -15
  102. lightning_sdk/lightning_cloud/openapi/models/id_index_body2.py +123 -0
  103. lightning_sdk/lightning_cloud/openapi/models/id_index_body3.py +175 -0
  104. lightning_sdk/lightning_cloud/openapi/models/id_reportlogsactivity_body.py +123 -0
  105. lightning_sdk/lightning_cloud/openapi/models/id_start_body.py +29 -3
  106. lightning_sdk/lightning_cloud/openapi/models/id_storage_body.py +175 -0
  107. lightning_sdk/lightning_cloud/openapi/models/id_uploads_body1.py +175 -0
  108. lightning_sdk/lightning_cloud/openapi/models/id_visibility_body.py +123 -0
  109. lightning_sdk/lightning_cloud/openapi/models/jobs_id_body1.py +27 -1
  110. lightning_sdk/lightning_cloud/openapi/models/jobs_id_body2.py +17 -43
  111. lightning_sdk/lightning_cloud/openapi/models/jobs_id_body3.py +149 -0
  112. lightning_sdk/lightning_cloud/openapi/models/litloggermetrics_id_body.py +175 -0
  113. lightning_sdk/lightning_cloud/openapi/models/litpages_id_body.py +27 -1
  114. lightning_sdk/lightning_cloud/openapi/models/loggermetrics_id_body.py +123 -0
  115. lightning_sdk/lightning_cloud/openapi/models/metrics_stream_id_loggerartifacts_body.py +123 -0
  116. lightning_sdk/lightning_cloud/openapi/models/metricsstream_create_body.py +383 -0
  117. lightning_sdk/lightning_cloud/openapi/models/{v1_cloud_space_id_list.py → metricsstream_delete_body.py} +10 -10
  118. lightning_sdk/lightning_cloud/openapi/models/metricsstream_id_body.py +175 -0
  119. lightning_sdk/lightning_cloud/openapi/models/{v1_upload_model_response.py → model_id_versions_body.py} +25 -51
  120. lightning_sdk/lightning_cloud/openapi/models/model_id_visibility_body.py +123 -0
  121. lightning_sdk/lightning_cloud/openapi/models/models_model_id_body.py +149 -0
  122. lightning_sdk/lightning_cloud/openapi/models/multimachinejobs_id_body.py +123 -0
  123. lightning_sdk/lightning_cloud/openapi/models/org_id_memberships_body.py +27 -1
  124. lightning_sdk/lightning_cloud/openapi/models/orgs_id_body.py +183 -1
  125. lightning_sdk/lightning_cloud/openapi/models/profiler_captures_body.py +279 -0
  126. lightning_sdk/lightning_cloud/openapi/models/profiler_enabled_body.py +149 -0
  127. lightning_sdk/lightning_cloud/openapi/models/project_id_agentmanagedendpoints_body.py +149 -0
  128. lightning_sdk/lightning_cloud/openapi/models/{project_id_assistants_body.py → project_id_agents_body.py} +181 -51
  129. lightning_sdk/lightning_cloud/openapi/models/project_id_cloudspaces_body.py +71 -19
  130. lightning_sdk/lightning_cloud/openapi/models/project_id_fileendpoints_body.py +79 -1
  131. lightning_sdk/lightning_cloud/openapi/models/project_id_jobs_body.py +175 -0
  132. lightning_sdk/lightning_cloud/openapi/models/project_id_litregistry_body.py +123 -0
  133. lightning_sdk/lightning_cloud/openapi/models/project_id_memberships_body.py +53 -1
  134. lightning_sdk/lightning_cloud/openapi/models/{v1_upload_model_request.py → project_id_models_body.py} +70 -70
  135. lightning_sdk/lightning_cloud/openapi/models/project_id_multimachinejobs_body.py +227 -0
  136. lightning_sdk/lightning_cloud/openapi/models/project_id_secrets_body.py +27 -1
  137. lightning_sdk/lightning_cloud/openapi/models/project_id_serviceexecution_body.py +175 -0
  138. lightning_sdk/lightning_cloud/openapi/models/project_id_snowflake_body.py +123 -0
  139. lightning_sdk/lightning_cloud/openapi/models/projects_id_body.py +29 -3
  140. lightning_sdk/lightning_cloud/openapi/models/query_query_id_body.py +175 -0
  141. lightning_sdk/lightning_cloud/openapi/models/secrets_id_body1.py +123 -0
  142. lightning_sdk/lightning_cloud/openapi/models/servers_server_id_body.py +123 -0
  143. lightning_sdk/lightning_cloud/openapi/models/service_artifact_artifact_kind.py +104 -0
  144. lightning_sdk/lightning_cloud/openapi/models/serviceexecution_id_body.py +331 -0
  145. lightning_sdk/lightning_cloud/openapi/models/slurm_jobs_body.py +79 -1
  146. lightning_sdk/lightning_cloud/openapi/models/snowflake_export_body.py +305 -0
  147. lightning_sdk/lightning_cloud/openapi/models/{v1_download_model_response.py → snowflake_query_body.py} +51 -51
  148. lightning_sdk/lightning_cloud/openapi/models/storage_complete_body.py +27 -1
  149. lightning_sdk/lightning_cloud/openapi/models/update.py +105 -1
  150. lightning_sdk/lightning_cloud/openapi/models/upload_id_complete_body.py +149 -0
  151. lightning_sdk/lightning_cloud/openapi/models/upload_id_parts_body.py +149 -0
  152. lightning_sdk/lightning_cloud/openapi/models/user_id_affiliatelinks_body.py +149 -0
  153. lightning_sdk/lightning_cloud/openapi/models/v1_accelerator_quota_info.py +201 -0
  154. lightning_sdk/lightning_cloud/openapi/models/v1_ack_user_storage_violation_response.py +97 -0
  155. lightning_sdk/lightning_cloud/openapi/models/v1_add_job_timing_response.py +97 -0
  156. lightning_sdk/lightning_cloud/openapi/models/v1_affiliate_link.py +435 -0
  157. lightning_sdk/lightning_cloud/openapi/models/v1_agent_job.py +131 -1
  158. lightning_sdk/lightning_cloud/openapi/models/v1_api_pricing_spec.py +149 -0
  159. lightning_sdk/lightning_cloud/openapi/models/v1_app_type.py +104 -0
  160. lightning_sdk/lightning_cloud/openapi/models/v1_append_logger_metrics_response.py +97 -0
  161. lightning_sdk/lightning_cloud/openapi/models/v1_approve_auto_join_domain_response.py +123 -0
  162. lightning_sdk/lightning_cloud/openapi/models/v1_assign_variant_response.py +97 -0
  163. lightning_sdk/lightning_cloud/openapi/models/v1_assistant.py +157 -1
  164. lightning_sdk/lightning_cloud/openapi/models/v1_assistant_knowledge_item_status.py +253 -0
  165. lightning_sdk/lightning_cloud/openapi/models/v1_assistant_knowledge_status.py +123 -0
  166. lightning_sdk/lightning_cloud/openapi/models/v1_assistant_model_status.py +104 -0
  167. lightning_sdk/lightning_cloud/openapi/models/v1_auto_join_domain_validation.py +175 -0
  168. lightning_sdk/lightning_cloud/openapi/models/v1_auto_join_org_response.py +149 -0
  169. lightning_sdk/lightning_cloud/openapi/models/v1_autoscaling_spec.py +305 -0
  170. lightning_sdk/lightning_cloud/openapi/models/v1_autoscaling_target_metric.py +149 -0
  171. lightning_sdk/lightning_cloud/openapi/models/v1_aws_direct_v1.py +133 -3
  172. lightning_sdk/lightning_cloud/openapi/models/v1_batch_update_lightningwork_response.py +97 -0
  173. lightning_sdk/lightning_cloud/openapi/models/v1_body.py +123 -0
  174. lightning_sdk/lightning_cloud/openapi/models/v1_cancellation_metadata.py +149 -0
  175. lightning_sdk/lightning_cloud/openapi/models/v1_capacity_block_offering.py +383 -0
  176. lightning_sdk/lightning_cloud/openapi/models/v1_check_snowflake_connection_response.py +123 -0
  177. lightning_sdk/lightning_cloud/openapi/models/v1_checkbox.py +201 -0
  178. lightning_sdk/lightning_cloud/openapi/models/v1_cloud_space.py +162 -6
  179. lightning_sdk/lightning_cloud/openapi/models/v1_cloud_space_code_version.py +383 -0
  180. lightning_sdk/lightning_cloud/openapi/models/v1_cloud_space_code_version_status.py +105 -0
  181. lightning_sdk/lightning_cloud/openapi/models/v1_cloud_space_instance_config.py +1 -53
  182. lightning_sdk/lightning_cloud/openapi/models/v1_cloud_space_instance_startup_status.py +79 -1
  183. lightning_sdk/lightning_cloud/openapi/models/{v1_cluster_log_service.py → v1_cloud_space_session.py} +49 -49
  184. lightning_sdk/lightning_cloud/openapi/models/v1_cloud_space_version.py +669 -0
  185. lightning_sdk/lightning_cloud/openapi/models/v1_cloud_space_version_publication.py +201 -0
  186. lightning_sdk/lightning_cloud/openapi/models/v1_cluster_accelerator.py +445 -3
  187. lightning_sdk/lightning_cloud/openapi/models/v1_cluster_capacity_reservation.py +55 -3
  188. lightning_sdk/lightning_cloud/openapi/models/v1_cluster_names.py +123 -0
  189. lightning_sdk/lightning_cloud/openapi/models/v1_cluster_proxy.py +201 -0
  190. lightning_sdk/lightning_cloud/openapi/models/v1_cluster_resource_tag.py +149 -0
  191. lightning_sdk/lightning_cloud/openapi/models/v1_cluster_security_options.py +357 -0
  192. lightning_sdk/lightning_cloud/openapi/models/v1_cluster_spec.py +157 -107
  193. lightning_sdk/lightning_cloud/openapi/models/v1_cluster_state.py +1 -0
  194. lightning_sdk/lightning_cloud/openapi/models/v1_cluster_status.py +17 -43
  195. lightning_sdk/lightning_cloud/openapi/models/v1_cluster_tagging_options.py +175 -0
  196. lightning_sdk/lightning_cloud/openapi/models/v1_cluster_type.py +0 -1
  197. lightning_sdk/lightning_cloud/openapi/models/v1_command_argument.py +131 -1
  198. lightning_sdk/lightning_cloud/openapi/models/v1_complete_model_upload_response.py +97 -0
  199. lightning_sdk/lightning_cloud/openapi/models/v1_complete_multi_part_upload_response.py +97 -0
  200. lightning_sdk/lightning_cloud/openapi/models/v1_complete_upload_service_execution_artifact_response.py +97 -0
  201. lightning_sdk/lightning_cloud/openapi/models/v1_completed_part.py +149 -0
  202. lightning_sdk/lightning_cloud/openapi/models/v1_conversation.py +15 -15
  203. lightning_sdk/lightning_cloud/openapi/models/v1_conversation_response_chunk.py +26 -26
  204. lightning_sdk/lightning_cloud/openapi/models/v1_count_metrics_streams_response.py +123 -0
  205. lightning_sdk/lightning_cloud/openapi/models/v1_create_checkout_session_request.py +53 -1
  206. lightning_sdk/lightning_cloud/openapi/models/v1_create_cluster_request.py +27 -1
  207. lightning_sdk/lightning_cloud/openapi/models/v1_create_deployment_template_request.py +539 -0
  208. lightning_sdk/lightning_cloud/openapi/models/v1_create_multi_part_upload_response.py +123 -0
  209. lightning_sdk/lightning_cloud/openapi/models/v1_create_organization_request.py +159 -3
  210. lightning_sdk/lightning_cloud/openapi/models/v1_create_project_request.py +27 -1
  211. lightning_sdk/lightning_cloud/openapi/models/v1_create_shared_metrics_stream_request.py +201 -0
  212. lightning_sdk/lightning_cloud/openapi/models/v1_create_shared_metrics_stream_response.py +123 -0
  213. lightning_sdk/lightning_cloud/openapi/models/v1_create_snowflake_connection_response.py +123 -0
  214. lightning_sdk/lightning_cloud/openapi/models/v1_create_user_secret_request.py +149 -0
  215. lightning_sdk/lightning_cloud/openapi/models/v1_data_connection.py +365 -1
  216. lightning_sdk/lightning_cloud/openapi/models/{v1_cluster_performance_profile.py → v1_data_connection_state.py} +11 -9
  217. lightning_sdk/lightning_cloud/openapi/models/v1_data_path.py +201 -0
  218. lightning_sdk/lightning_cloud/openapi/models/v1_delete_affiliate_link_response.py +97 -0
  219. lightning_sdk/lightning_cloud/openapi/models/v1_delete_cloud_space_session_response.py +97 -0
  220. lightning_sdk/lightning_cloud/openapi/models/v1_delete_cloud_space_version_publication_response.py +97 -0
  221. lightning_sdk/lightning_cloud/openapi/models/v1_delete_cloud_space_version_response.py +97 -0
  222. lightning_sdk/lightning_cloud/openapi/models/v1_delete_cluster_proxy_response.py +97 -0
  223. lightning_sdk/lightning_cloud/openapi/models/v1_delete_deployment_release_response.py +97 -0
  224. lightning_sdk/lightning_cloud/openapi/models/{v1_azure_cluster_driver_status.py → v1_delete_deployment_response.py} +6 -6
  225. lightning_sdk/lightning_cloud/openapi/models/v1_delete_index_response.py +97 -0
  226. lightning_sdk/lightning_cloud/openapi/models/{v1_byom_cluster_driver.py → v1_delete_job_response.py} +6 -6
  227. lightning_sdk/lightning_cloud/openapi/models/{v1_delete_lightningapp_v2_response.py → v1_delete_logger_artifact_response.py} +6 -6
  228. lightning_sdk/lightning_cloud/openapi/models/v1_delete_managed_endpoint_response.py +97 -0
  229. lightning_sdk/lightning_cloud/openapi/models/v1_delete_metrics_stream_response.py +97 -0
  230. lightning_sdk/lightning_cloud/openapi/models/v1_delete_multi_machine_job_response.py +97 -0
  231. lightning_sdk/lightning_cloud/openapi/models/v1_delete_profiler_capture_response.py +97 -0
  232. lightning_sdk/lightning_cloud/openapi/models/v1_delete_service_execution_response.py +97 -0
  233. lightning_sdk/lightning_cloud/openapi/models/{v1_delete_lightningapp_release_response.py → v1_delete_shared_metrics_stream_response.py} +6 -6
  234. lightning_sdk/lightning_cloud/openapi/models/v1_deployment.py +617 -0
  235. lightning_sdk/lightning_cloud/openapi/models/v1_deployment_api.py +227 -0
  236. lightning_sdk/lightning_cloud/openapi/models/v1_deployment_event.py +357 -0
  237. lightning_sdk/lightning_cloud/openapi/models/v1_deployment_event_type.py +104 -0
  238. lightning_sdk/lightning_cloud/openapi/models/v1_deployment_metrics.py +149 -0
  239. lightning_sdk/lightning_cloud/openapi/models/v1_deployment_performance.py +305 -0
  240. lightning_sdk/lightning_cloud/openapi/models/v1_deployment_release.py +331 -0
  241. lightning_sdk/lightning_cloud/openapi/models/v1_deployment_spec.py +201 -0
  242. lightning_sdk/lightning_cloud/openapi/models/v1_deployment_state.py +105 -0
  243. lightning_sdk/lightning_cloud/openapi/models/v1_deployment_status.py +279 -0
  244. lightning_sdk/lightning_cloud/openapi/models/{appsv2_id_body.py → v1_deployment_strategy.py} +39 -39
  245. lightning_sdk/lightning_cloud/openapi/models/v1_deployment_template.py +721 -0
  246. lightning_sdk/lightning_cloud/openapi/models/v1_deployment_template_engagement_response.py +97 -0
  247. lightning_sdk/lightning_cloud/openapi/models/v1_deployment_template_gallery_response.py +591 -0
  248. lightning_sdk/lightning_cloud/openapi/models/v1_deployment_template_parameter.py +435 -0
  249. lightning_sdk/lightning_cloud/openapi/models/v1_deployment_template_parameter_placement.py +106 -0
  250. lightning_sdk/lightning_cloud/openapi/models/v1_deployment_template_parameter_type.py +106 -0
  251. lightning_sdk/lightning_cloud/openapi/models/v1_deployment_template_summary.py +591 -0
  252. lightning_sdk/lightning_cloud/openapi/models/{instance_type_availability.py → v1_deployment_template_type.py} +10 -10
  253. lightning_sdk/lightning_cloud/openapi/models/{v1_get_lightningapp_source_code_download_url_response.py → v1_download_job_logs_response.py} +10 -10
  254. lightning_sdk/lightning_cloud/openapi/models/v1_download_service_execution_artifact_response.py +175 -0
  255. lightning_sdk/lightning_cloud/openapi/models/v1_ebs.py +279 -0
  256. lightning_sdk/lightning_cloud/openapi/models/v1_efs_config.py +201 -0
  257. lightning_sdk/lightning_cloud/openapi/models/v1_endpoint.py +53 -27
  258. lightning_sdk/lightning_cloud/openapi/models/v1_endpoint_auth.py +27 -1
  259. lightning_sdk/lightning_cloud/openapi/models/v1_endpoint_type.py +104 -0
  260. lightning_sdk/lightning_cloud/openapi/models/v1_execute_cloud_space_command_response.py +29 -3
  261. lightning_sdk/lightning_cloud/openapi/models/v1_execute_in_cloud_space_session_response.py +97 -0
  262. lightning_sdk/lightning_cloud/openapi/models/v1_execute_snowflake_query_response.py +149 -0
  263. lightning_sdk/lightning_cloud/openapi/models/v1_experiment.py +409 -0
  264. lightning_sdk/lightning_cloud/openapi/models/v1_export_snowflake_query_response.py +123 -0
  265. lightning_sdk/lightning_cloud/openapi/models/v1_external_search_user.py +279 -0
  266. lightning_sdk/lightning_cloud/openapi/models/v1_file_endpoint.py +131 -1
  267. lightning_sdk/lightning_cloud/openapi/models/{v1_instance_spec.py → v1_filesystem_app.py} +85 -59
  268. lightning_sdk/lightning_cloud/openapi/models/v1_filesystem_cloud_space.py +149 -0
  269. lightning_sdk/lightning_cloud/openapi/models/v1_filesystem_dataset.py +123 -0
  270. lightning_sdk/lightning_cloud/openapi/models/v1_filesystem_job.py +175 -0
  271. lightning_sdk/lightning_cloud/openapi/models/v1_filesystem_slurm_job.py +149 -0
  272. lightning_sdk/lightning_cloud/openapi/models/v1_filesystem_snowflake_connection.py +123 -0
  273. lightning_sdk/lightning_cloud/openapi/models/v1_filesystem_work.py +149 -0
  274. lightning_sdk/lightning_cloud/openapi/models/v1_find_capacity_block_offering_response.py +123 -0
  275. lightning_sdk/lightning_cloud/openapi/models/v1_gcp_data_connection.py +27 -1
  276. lightning_sdk/lightning_cloud/openapi/models/v1_gcs_folder_data_connection.py +123 -0
  277. lightning_sdk/lightning_cloud/openapi/models/v1_get_affiliate_link_response.py +123 -0
  278. lightning_sdk/lightning_cloud/openapi/models/{v1_list_cluster_instance_types_response.py → v1_get_deployment_routing_telemetry_aggregated_response.py} +23 -23
  279. lightning_sdk/lightning_cloud/openapi/models/v1_get_deployment_routing_telemetry_response.py +123 -0
  280. lightning_sdk/lightning_cloud/openapi/models/v1_get_folder_index_response.py +27 -1
  281. lightning_sdk/lightning_cloud/openapi/models/{v1_cluster_driver_status.py → v1_get_job_stats_response.py} +39 -39
  282. lightning_sdk/lightning_cloud/openapi/models/v1_get_job_system_metrics_response.py +123 -0
  283. lightning_sdk/lightning_cloud/openapi/models/v1_get_lightningapp_instance_open_ports_response.py +123 -0
  284. lightning_sdk/lightning_cloud/openapi/models/v1_get_logger_metrics_response.py +123 -0
  285. lightning_sdk/lightning_cloud/openapi/models/v1_get_model_file_upload_urls_response.py +123 -0
  286. lightning_sdk/lightning_cloud/openapi/models/v1_get_model_file_url_response.py +175 -0
  287. lightning_sdk/lightning_cloud/openapi/models/v1_get_model_files_response.py +279 -0
  288. lightning_sdk/lightning_cloud/openapi/models/v1_get_model_files_url_response.py +149 -0
  289. lightning_sdk/lightning_cloud/openapi/models/{v1_lightningwork_cluster_driver.py → v1_get_project_artifact_response.py} +33 -14
  290. lightning_sdk/lightning_cloud/openapi/models/v1_get_project_balance_response.py +27 -1
  291. lightning_sdk/lightning_cloud/openapi/models/v1_get_project_storage_metadata_response.py +383 -0
  292. lightning_sdk/lightning_cloud/openapi/models/v1_get_service_execution_status_response.py +175 -0
  293. lightning_sdk/lightning_cloud/openapi/models/v1_get_snowflake_query_response.py +149 -0
  294. lightning_sdk/lightning_cloud/openapi/models/v1_get_user_balance_response.py +27 -1
  295. lightning_sdk/lightning_cloud/openapi/models/v1_get_user_response.py +105 -1
  296. lightning_sdk/lightning_cloud/openapi/models/v1_get_user_storage_breakdown_response.py +279 -0
  297. lightning_sdk/lightning_cloud/openapi/models/v1_get_user_storage_response.py +201 -0
  298. lightning_sdk/lightning_cloud/openapi/models/v1_google_cloud_direct_v1.py +163 -3
  299. lightning_sdk/lightning_cloud/openapi/models/v1_google_cloud_direct_v1_status.py +123 -0
  300. lightning_sdk/lightning_cloud/openapi/models/v1_header.py +175 -0
  301. lightning_sdk/lightning_cloud/openapi/models/{v1_external_kubeconfig.py → v1_health_check_exec.py} +21 -21
  302. lightning_sdk/lightning_cloud/openapi/models/v1_health_check_http_get.py +149 -0
  303. lightning_sdk/lightning_cloud/openapi/models/v1_ids_logger_metrics.py +123 -0
  304. lightning_sdk/lightning_cloud/openapi/models/v1_input.py +175 -0
  305. lightning_sdk/lightning_cloud/openapi/models/v1_interrupt_server_response.py +97 -0
  306. lightning_sdk/lightning_cloud/openapi/models/v1_job.py +697 -0
  307. lightning_sdk/lightning_cloud/openapi/models/v1_job_health_check_config.py +253 -0
  308. lightning_sdk/lightning_cloud/openapi/models/v1_job_log_entry.py +175 -0
  309. lightning_sdk/lightning_cloud/openapi/models/v1_job_logs_page.py +227 -0
  310. lightning_sdk/lightning_cloud/openapi/models/v1_job_logs_response.py +149 -0
  311. lightning_sdk/lightning_cloud/openapi/models/v1_job_spec.py +669 -0
  312. lightning_sdk/lightning_cloud/openapi/models/v1_job_timing.py +201 -0
  313. lightning_sdk/lightning_cloud/openapi/models/v1_joinable_organization.py +331 -0
  314. lightning_sdk/lightning_cloud/openapi/models/v1_knowledge_configuration.py +279 -0
  315. lightning_sdk/lightning_cloud/openapi/models/v1_lambda_labs_direct_v1.py +125 -0
  316. lightning_sdk/lightning_cloud/openapi/models/v1_lightning_run.py +27 -1
  317. lightning_sdk/lightning_cloud/openapi/models/v1_lightningapp_instance_spec.py +105 -27
  318. lightning_sdk/lightning_cloud/openapi/models/v1_lightningapp_instance_status.py +53 -1
  319. lightning_sdk/lightning_cloud/openapi/models/v1_lightningwork_spec.py +27 -27
  320. lightning_sdk/lightning_cloud/openapi/models/v1_lightningwork_status.py +81 -3
  321. lightning_sdk/lightning_cloud/openapi/models/v1_list_affiliate_links_response.py +123 -0
  322. lightning_sdk/lightning_cloud/openapi/models/v1_list_cloud_space_python_versions_response.py +123 -0
  323. lightning_sdk/lightning_cloud/openapi/models/v1_list_cloud_space_sessions_response.py +123 -0
  324. lightning_sdk/lightning_cloud/openapi/models/v1_list_cloud_space_tags_response.py +6 -6
  325. lightning_sdk/lightning_cloud/openapi/models/v1_list_cloud_space_version_publications_response.py +123 -0
  326. lightning_sdk/lightning_cloud/openapi/models/v1_list_cloud_space_versions_response.py +123 -0
  327. lightning_sdk/lightning_cloud/openapi/models/v1_list_cluster_availabilities_response.py +123 -0
  328. lightning_sdk/lightning_cloud/openapi/models/v1_list_cluster_proxies_response.py +123 -0
  329. lightning_sdk/lightning_cloud/openapi/models/v1_list_deployment_events_response.py +123 -0
  330. lightning_sdk/lightning_cloud/openapi/models/v1_list_deployment_releases_response.py +123 -0
  331. lightning_sdk/lightning_cloud/openapi/models/v1_list_deployment_tags_response.py +123 -0
  332. lightning_sdk/lightning_cloud/openapi/models/v1_list_deployment_templates_response.py +175 -0
  333. lightning_sdk/lightning_cloud/openapi/models/v1_list_deployments_response.py +123 -0
  334. lightning_sdk/lightning_cloud/openapi/models/v1_list_experiments_response.py +149 -0
  335. lightning_sdk/lightning_cloud/openapi/models/v1_list_filesystem_apps_response.py +123 -0
  336. lightning_sdk/lightning_cloud/openapi/models/v1_list_filesystem_cloud_spaces_response.py +123 -0
  337. lightning_sdk/lightning_cloud/openapi/models/v1_list_filesystem_datasets_response.py +123 -0
  338. lightning_sdk/lightning_cloud/openapi/models/v1_list_filesystem_jobs_response.py +123 -0
  339. lightning_sdk/lightning_cloud/openapi/models/v1_list_filesystem_slurm_jobs_response.py +123 -0
  340. lightning_sdk/lightning_cloud/openapi/models/v1_list_filesystem_snowflake_response.py +123 -0
  341. lightning_sdk/lightning_cloud/openapi/models/{v1_list_lightningapps_v2_response.py → v1_list_jobs_response.py} +31 -31
  342. lightning_sdk/lightning_cloud/openapi/models/v1_list_joinable_organizations_response.py +123 -0
  343. lightning_sdk/lightning_cloud/openapi/models/v1_list_logger_artifact_response.py +123 -0
  344. lightning_sdk/lightning_cloud/openapi/models/v1_list_metrics_streams_response.py +123 -0
  345. lightning_sdk/lightning_cloud/openapi/models/v1_list_multi_machine_job_events_response.py +123 -0
  346. lightning_sdk/lightning_cloud/openapi/models/v1_list_multi_machine_jobs_response.py +123 -0
  347. lightning_sdk/lightning_cloud/openapi/models/v1_list_new_features_for_user_response.py +123 -0
  348. lightning_sdk/lightning_cloud/openapi/models/v1_list_profiler_captures_response.py +123 -0
  349. lightning_sdk/lightning_cloud/openapi/models/v1_list_project_locked_resources_response.py +123 -0
  350. lightning_sdk/lightning_cloud/openapi/models/v1_list_published_cloud_spaces_response.py +3 -29
  351. lightning_sdk/lightning_cloud/openapi/models/v1_list_published_deployment_templates_response.py +175 -0
  352. lightning_sdk/lightning_cloud/openapi/models/v1_list_service_execution_lightningapp_instances_response.py +175 -0
  353. lightning_sdk/lightning_cloud/openapi/models/v1_list_service_executions_response.py +123 -0
  354. lightning_sdk/lightning_cloud/openapi/models/v1_lit_page.py +27 -1
  355. lightning_sdk/lightning_cloud/openapi/models/v1_lit_registry_project.py +227 -0
  356. lightning_sdk/lightning_cloud/openapi/models/v1_lit_repository.py +279 -0
  357. lightning_sdk/lightning_cloud/openapi/models/v1_locked_resource.py +227 -0
  358. lightning_sdk/lightning_cloud/openapi/models/v1_logger_artifact.py +227 -0
  359. lightning_sdk/lightning_cloud/openapi/models/v1_machines_selector.py +149 -0
  360. lightning_sdk/lightning_cloud/openapi/models/v1_magic_link_login_request.py +81 -3
  361. lightning_sdk/lightning_cloud/openapi/models/v1_magic_link_login_response.py +53 -1
  362. lightning_sdk/lightning_cloud/openapi/models/v1_managed_endpoint.py +175 -19
  363. lightning_sdk/lightning_cloud/openapi/models/v1_managed_model.py +305 -0
  364. lightning_sdk/lightning_cloud/openapi/models/v1_managed_model_abilities.py +175 -0
  365. lightning_sdk/lightning_cloud/openapi/models/v1_membership.py +79 -1
  366. lightning_sdk/lightning_cloud/openapi/models/v1_message.py +138 -8
  367. lightning_sdk/lightning_cloud/openapi/models/v1_message_content.py +6 -6
  368. lightning_sdk/lightning_cloud/openapi/models/v1_message_content_type.py +103 -0
  369. lightning_sdk/lightning_cloud/openapi/models/v1_metadata.py +131 -1
  370. lightning_sdk/lightning_cloud/openapi/models/v1_metric_value.py +175 -0
  371. lightning_sdk/lightning_cloud/openapi/models/v1_metrics.py +175 -0
  372. lightning_sdk/lightning_cloud/openapi/models/v1_metrics_stream.py +799 -0
  373. lightning_sdk/lightning_cloud/openapi/models/v1_metrics_tags.py +201 -0
  374. lightning_sdk/lightning_cloud/openapi/models/v1_metrics_tracker.py +383 -0
  375. lightning_sdk/lightning_cloud/openapi/models/v1_model.py +95 -17
  376. lightning_sdk/lightning_cloud/openapi/models/v1_model_file.py +175 -0
  377. lightning_sdk/lightning_cloud/openapi/models/v1_model_version_archive.py +131 -1
  378. lightning_sdk/lightning_cloud/openapi/models/v1_mount_target.py +201 -0
  379. lightning_sdk/lightning_cloud/openapi/models/v1_multi_machine_job.py +487 -0
  380. lightning_sdk/lightning_cloud/openapi/models/v1_multi_machine_job_event.py +331 -0
  381. lightning_sdk/lightning_cloud/openapi/models/{get_cluster_health_response_health_status.py → v1_multi_machine_job_event_type.py} +9 -9
  382. lightning_sdk/lightning_cloud/openapi/models/v1_multi_machine_job_fault_tolerance.py +149 -0
  383. lightning_sdk/lightning_cloud/openapi/models/v1_multi_machine_job_fault_tolerance_strategy.py +105 -0
  384. lightning_sdk/lightning_cloud/openapi/models/v1_multi_machine_job_state.py +108 -0
  385. lightning_sdk/lightning_cloud/openapi/models/v1_multi_machine_job_status.py +305 -0
  386. lightning_sdk/lightning_cloud/openapi/models/v1_named_get_logger_metrics.py +123 -0
  387. lightning_sdk/lightning_cloud/openapi/models/v1_new_feature.py +383 -0
  388. lightning_sdk/lightning_cloud/openapi/models/v1_onboarding_event_request.py +175 -0
  389. lightning_sdk/lightning_cloud/openapi/models/v1_onboarding_event_response.py +97 -0
  390. lightning_sdk/lightning_cloud/openapi/models/v1_organization.py +209 -1
  391. lightning_sdk/lightning_cloud/openapi/models/v1_parameterization_spec.py +227 -0
  392. lightning_sdk/lightning_cloud/openapi/models/v1_path_telemetry.py +123 -0
  393. lightning_sdk/lightning_cloud/openapi/models/v1_phase_type.py +104 -0
  394. lightning_sdk/lightning_cloud/openapi/models/v1_profiler_capture.py +357 -0
  395. lightning_sdk/lightning_cloud/openapi/models/v1_profiler_enabled_response.py +123 -0
  396. lightning_sdk/lightning_cloud/openapi/models/v1_project.py +131 -1
  397. lightning_sdk/lightning_cloud/openapi/models/v1_project_membership.py +263 -3
  398. lightning_sdk/lightning_cloud/openapi/models/v1_project_settings.py +29 -3
  399. lightning_sdk/lightning_cloud/openapi/models/v1_project_storage.py +279 -0
  400. lightning_sdk/lightning_cloud/openapi/models/v1_published_cloud_space_response.py +188 -6
  401. lightning_sdk/lightning_cloud/openapi/models/v1_purchase_capacity_block_response.py +175 -0
  402. lightning_sdk/lightning_cloud/openapi/models/v1_query_param.py +175 -0
  403. lightning_sdk/lightning_cloud/openapi/models/v1_query_result.py +123 -0
  404. lightning_sdk/lightning_cloud/openapi/models/v1_query_result_row.py +123 -0
  405. lightning_sdk/lightning_cloud/openapi/models/v1_refresh_path_response.py +97 -0
  406. lightning_sdk/lightning_cloud/openapi/models/v1_regional_load_balancer.py +149 -0
  407. lightning_sdk/lightning_cloud/openapi/models/v1_report_logs_activity_response.py +97 -0
  408. lightning_sdk/lightning_cloud/openapi/models/v1_request_cluster_access_request.py +175 -0
  409. lightning_sdk/lightning_cloud/openapi/models/v1_request_cluster_access_response.py +97 -0
  410. lightning_sdk/lightning_cloud/openapi/models/{v1_cloud_space_tag.py → v1_resource_tag.py} +18 -18
  411. lightning_sdk/lightning_cloud/openapi/models/v1_resource_visibility.py +123 -0
  412. lightning_sdk/lightning_cloud/openapi/models/v1_resources.py +37 -11
  413. lightning_sdk/lightning_cloud/openapi/models/v1_restore_deployment_release_response.py +97 -0
  414. lightning_sdk/lightning_cloud/openapi/models/v1_rolling_update_strategy.py +149 -0
  415. lightning_sdk/lightning_cloud/openapi/models/v1_rule_condition.py +29 -3
  416. lightning_sdk/lightning_cloud/openapi/models/v1_rule_resource.py +11 -0
  417. lightning_sdk/lightning_cloud/openapi/models/v1_s3_folder_data_connection.py +123 -0
  418. lightning_sdk/lightning_cloud/openapi/models/v1_search_job_logs_response.py +149 -0
  419. lightning_sdk/lightning_cloud/openapi/models/v1_search_users_response.py +6 -6
  420. lightning_sdk/lightning_cloud/openapi/models/v1_secret.py +107 -3
  421. lightning_sdk/lightning_cloud/openapi/models/{v1_lightningapp_cluster_driver.py → v1_secret_type.py} +10 -9
  422. lightning_sdk/lightning_cloud/openapi/models/v1_select.py +149 -0
  423. lightning_sdk/lightning_cloud/openapi/models/v1_server_check_in_response.py +97 -0
  424. lightning_sdk/lightning_cloud/openapi/models/{v1_update_cluster_response.py → v1_service_artifact.py} +54 -54
  425. lightning_sdk/lightning_cloud/openapi/models/v1_service_execution.py +383 -0
  426. lightning_sdk/lightning_cloud/openapi/models/v1_should_start_syncing_response.py +123 -0
  427. lightning_sdk/lightning_cloud/openapi/models/v1_signed_url.py +149 -0
  428. lightning_sdk/lightning_cloud/openapi/models/v1_slurm_job.py +131 -1
  429. lightning_sdk/lightning_cloud/openapi/models/v1_slurm_node.py +53 -1
  430. lightning_sdk/lightning_cloud/openapi/models/v1_slurm_v1.py +27 -1
  431. lightning_sdk/lightning_cloud/openapi/models/v1_snowflake_data_connection.py +253 -0
  432. lightning_sdk/lightning_cloud/openapi/models/v1_status_code_telemetry.py +123 -0
  433. lightning_sdk/lightning_cloud/openapi/models/v1_storage_asset.py +253 -0
  434. lightning_sdk/lightning_cloud/openapi/models/v1_storage_asset_type.py +105 -0
  435. lightning_sdk/lightning_cloud/openapi/models/v1_studio_job_app.py +1 -0
  436. lightning_sdk/lightning_cloud/openapi/models/v1_system_info.py +617 -0
  437. lightning_sdk/lightning_cloud/openapi/models/v1_telemetry.py +331 -0
  438. lightning_sdk/lightning_cloud/openapi/models/v1_timestamp_code_telemetry.py +123 -0
  439. lightning_sdk/lightning_cloud/openapi/models/v1_transaction.py +27 -1
  440. lightning_sdk/lightning_cloud/openapi/models/v1_transfer_user_balance_request.py +27 -1
  441. lightning_sdk/lightning_cloud/openapi/models/v1_update_billing_subscription_request.py +53 -1
  442. lightning_sdk/lightning_cloud/openapi/models/v1_update_cluster_accelerators_request.py +149 -0
  443. lightning_sdk/lightning_cloud/openapi/models/v1_update_index_response.py +97 -0
  444. lightning_sdk/lightning_cloud/openapi/models/v1_update_metrics_stream_visibility_response.py +97 -0
  445. lightning_sdk/lightning_cloud/openapi/models/v1_update_model_visibility_response.py +97 -0
  446. lightning_sdk/lightning_cloud/openapi/models/v1_update_shared_metrics_stream_response.py +97 -0
  447. lightning_sdk/lightning_cloud/openapi/models/v1_update_snowflake_query_response.py +97 -0
  448. lightning_sdk/lightning_cloud/openapi/models/v1_update_user_request.py +53 -1
  449. lightning_sdk/lightning_cloud/openapi/models/v1_update_user_viewed_new_features_request.py +123 -0
  450. lightning_sdk/lightning_cloud/openapi/models/v1_update_user_viewed_new_features_response.py +97 -0
  451. lightning_sdk/lightning_cloud/openapi/models/v1_upload_service_execution_artifact_parts_response.py +123 -0
  452. lightning_sdk/lightning_cloud/openapi/models/v1_upload_service_execution_artifact_response.py +149 -0
  453. lightning_sdk/lightning_cloud/openapi/models/v1_upstream_cloud_space.py +97 -19
  454. lightning_sdk/lightning_cloud/openapi/models/v1_upstream_job.py +227 -0
  455. lightning_sdk/lightning_cloud/openapi/models/v1_usage.py +27 -1
  456. lightning_sdk/lightning_cloud/openapi/models/v1_usage_details.py +107 -3
  457. lightning_sdk/lightning_cloud/openapi/models/v1_user_features.py +1030 -302
  458. lightning_sdk/lightning_cloud/openapi/models/v1_user_requested_compute_config.py +45 -45
  459. lightning_sdk/lightning_cloud/openapi/models/v1_user_requested_flow_compute_config.py +29 -29
  460. lightning_sdk/lightning_cloud/openapi/models/v1_validate_assistant_status_response.py +149 -0
  461. lightning_sdk/lightning_cloud/openapi/models/v1_validate_auto_join_domain_response.py +97 -0
  462. lightning_sdk/lightning_cloud/openapi/models/v1_validate_data_connection_response.py +107 -3
  463. lightning_sdk/lightning_cloud/openapi/models/v1_validate_deployment_image_request.py +149 -0
  464. lightning_sdk/lightning_cloud/openapi/models/v1_validate_deployment_image_response.py +97 -0
  465. lightning_sdk/lightning_cloud/openapi/models/v1_validate_managed_endpoint_request.py +175 -0
  466. lightning_sdk/lightning_cloud/openapi/models/v1_validate_managed_endpoint_response.py +123 -0
  467. lightning_sdk/lightning_cloud/openapi/models/v1_validate_managed_model_response.py +123 -0
  468. lightning_sdk/lightning_cloud/openapi/models/v1_volume.py +175 -0
  469. lightning_sdk/lightning_cloud/openapi/models/v1_vultr_direct_v1.py +125 -0
  470. lightning_sdk/lightning_cloud/openapi/models/validate.py +97 -19
  471. lightning_sdk/lightning_cloud/openapi/models/validateautojoindomain_domain_body.py +123 -0
  472. lightning_sdk/lightning_cloud/openapi/models/version_uploads_body.py +123 -0
  473. lightning_sdk/lightning_cloud/openapi/models/versions_id_body.py +357 -0
  474. lightning_sdk/lightning_cloud/rest_client.py +49 -23
  475. lightning_sdk/lightning_cloud/source_code/logs_socket_api.py +1 -1
  476. lightning_sdk/lightning_cloud/source_code/tar.py +1 -3
  477. lightning_sdk/lightning_cloud/utils/data_connection.py +145 -7
  478. lightning_sdk/machine.py +17 -4
  479. lightning_sdk/mmt/__init__.py +4 -0
  480. lightning_sdk/mmt/base.py +288 -0
  481. lightning_sdk/mmt/mmt.py +282 -0
  482. lightning_sdk/mmt/v1.py +185 -0
  483. lightning_sdk/mmt/v2.py +193 -0
  484. lightning_sdk/models.py +153 -0
  485. lightning_sdk/organization.py +1 -1
  486. lightning_sdk/plugin.py +207 -41
  487. lightning_sdk/services/__init__.py +5 -0
  488. lightning_sdk/services/file_endpoint.py +223 -0
  489. lightning_sdk/services/finetune/__init__.py +35 -0
  490. lightning_sdk/services/utilities.py +111 -0
  491. lightning_sdk/status.py +2 -1
  492. lightning_sdk/studio.py +90 -17
  493. lightning_sdk/teamspace.py +189 -11
  494. lightning_sdk/user.py +1 -1
  495. lightning_sdk/utils/__init__.py +0 -0
  496. lightning_sdk/utils/dynamic.py +61 -0
  497. lightning_sdk/utils/enum.py +116 -0
  498. lightning_sdk/{utils.py → utils/resolve.py} +41 -4
  499. lightning_sdk-0.1.46.dist-info/LICENSE +21 -0
  500. {lightning_sdk-0.1.2.dist-info → lightning_sdk-0.1.46.dist-info}/METADATA +30 -4
  501. lightning_sdk-0.1.46.dist-info/RECORD +858 -0
  502. {lightning_sdk-0.1.2.dist-info → lightning_sdk-0.1.46.dist-info}/WHEEL +1 -1
  503. lightning_sdk/lightning_cloud/openapi/models/app_id_releases_body.py +0 -541
  504. lightning_sdk/lightning_cloud/openapi/models/assistants_id_body.py +0 -487
  505. lightning_sdk/lightning_cloud/openapi/models/id_endpoint_body.py +0 -409
  506. lightning_sdk/lightning_cloud/openapi/models/id_get_body1.py +0 -333
  507. lightning_sdk/lightning_cloud/openapi/models/project_id_appsv2_body.py +0 -201
  508. lightning_sdk/lightning_cloud/openapi/models/v1_aws_cluster_driver_spec.py +0 -1039
  509. lightning_sdk/lightning_cloud/openapi/models/v1_aws_cluster_secondary_region_spec.py +0 -253
  510. lightning_sdk/lightning_cloud/openapi/models/v1_azure_cluster_driver_spec.py +0 -227
  511. lightning_sdk/lightning_cloud/openapi/models/v1_cluster_driver.py +0 -175
  512. lightning_sdk/lightning_cloud/openapi/models/v1_container_resources.py +0 -201
  513. lightning_sdk/lightning_cloud/openapi/models/v1_eks_custer_driver_status.py +0 -387
  514. lightning_sdk/lightning_cloud/openapi/models/v1_instance_type.py +0 -305
  515. lightning_sdk/lightning_cloud/openapi/models/v1_kubernetes_cluster_driver.py +0 -359
  516. lightning_sdk/lightning_cloud/openapi/models/v1_kubernetes_cluster_status.py +0 -279
  517. lightning_sdk/lightning_cloud/openapi/models/v1_lightningapp_release.py +0 -697
  518. lightning_sdk/lightning_cloud/openapi/models/v1_lightningapp_v2.py +0 -331
  519. lightning_sdk-0.1.2.dist-info/RECORD +0 -550
  520. {lightning_sdk-0.1.2.dist-info → lightning_sdk-0.1.46.dist-info}/entry_points.txt +0 -0
  521. {lightning_sdk-0.1.2.dist-info → lightning_sdk-0.1.46.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,282 @@
1
+ from contextlib import suppress
2
+ from functools import lru_cache
3
+ from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union
4
+
5
+ from lightning_sdk.api.user_api import UserApi
6
+ from lightning_sdk.job.job import _has_jobs_v2
7
+ from lightning_sdk.mmt.base import MMTMachine, _BaseMMT
8
+ from lightning_sdk.mmt.v1 import _MMTV1
9
+ from lightning_sdk.mmt.v2 import _MMTV2
10
+ from lightning_sdk.utils.resolve import _setup_logger
11
+
12
+ if TYPE_CHECKING:
13
+ from lightning_sdk.machine import Machine
14
+ from lightning_sdk.organization import Organization
15
+ from lightning_sdk.status import Status
16
+ from lightning_sdk.studio import Studio
17
+ from lightning_sdk.teamspace import Teamspace
18
+ from lightning_sdk.user import User
19
+
20
+ _logger = _setup_logger(__name__)
21
+
22
+
23
+ @lru_cache(maxsize=None)
24
+ def _has_mmt_v2() -> bool:
25
+ # users need both mmtv2 and jobsv2 flags in order for mmtv2 to work correctly
26
+ if not _has_jobs_v2():
27
+ return False
28
+
29
+ api = UserApi()
30
+ try:
31
+ return api._get_feature_flags().mmt_v2
32
+ except Exception:
33
+ return False
34
+
35
+
36
+ class MMT(_BaseMMT):
37
+ """Class to submit and manage multi-machine jobs on the Lightning AI Platform."""
38
+
39
+ _force_v1: (
40
+ bool
41
+ ) = False # required for studio plugin still working correctly as v2 currently does not support the studio env
42
+
43
+ def __init__(
44
+ self,
45
+ name: str,
46
+ teamspace: Union[str, "Teamspace", None] = None,
47
+ org: Union[str, "Organization", None] = None,
48
+ user: Union[str, "User", None] = None,
49
+ *,
50
+ _fetch_job: bool = True,
51
+ ) -> None:
52
+ """Fetch already existing jobs.
53
+
54
+ Args:
55
+ name: the name of the job
56
+ teamspace: the teamspace the job is part of
57
+ org: the name of the organization owning the :param`teamspace` in case it is owned by an org
58
+ user: the name of the user owning the :param`teamspace`
59
+ in case it is owned directly by a user instead of an org.
60
+ """
61
+ internal_mmt_cls = _MMTV2 if _has_mmt_v2() and not self._force_v1 else _MMTV1
62
+
63
+ self._internal_mmt = internal_mmt_cls(
64
+ name=name,
65
+ teamspace=teamspace,
66
+ org=org,
67
+ user=user,
68
+ _fetch_job=_fetch_job,
69
+ )
70
+
71
+ @classmethod
72
+ def run(
73
+ cls,
74
+ name: str,
75
+ num_machines: int,
76
+ machine: "Machine",
77
+ command: Optional[str] = None,
78
+ studio: Union["Studio", str, None] = None,
79
+ image: Union[str, None] = None,
80
+ teamspace: Union[str, "Teamspace", None] = None,
81
+ org: Union[str, "Organization", None] = None,
82
+ user: Union[str, "User", None] = None,
83
+ cloud_account: Optional[str] = None,
84
+ env: Optional[Dict[str, str]] = None,
85
+ interruptible: bool = False,
86
+ image_credentials: Optional[str] = None,
87
+ cloud_account_auth: bool = False,
88
+ artifacts_local: Optional[str] = None,
89
+ artifacts_remote: Optional[str] = None,
90
+ cluster: Optional[str] = None, # deprecated in favor of cloud_account
91
+ ) -> "MMT":
92
+ """Run async workloads using a docker image across multiple machines.
93
+
94
+ Args:
95
+ name: The name of the job. Needs to be unique within the teamspace.
96
+ machine: The machine type to run the job on. One of {", ".join(_MACHINE_VALUES)}.
97
+ num_machine: The number of machines to run on.
98
+ command: The command to run inside your job. Required if using a studio. Optional if using an image.
99
+ If not provided for images, will run the container entrypoint and default command.
100
+ studio: The studio env to run the job with. Mutually exclusive with image.
101
+ image: The docker image to run the job with. Mutually exclusive with studio.
102
+ teamspace: The teamspace the job should be associated with. Defaults to the current teamspace.
103
+ org: The organization owning the teamspace (if any). Defaults to the current organization.
104
+ user: The user owning the teamspace (if any). Defaults to the current user.
105
+ cloud_account: The cloud account to run the job on.
106
+ Defaults to the studio cloud account if running with studio compute env.
107
+ If not provided will fall back to the teamspaces default cloud account.
108
+ env: Environment variables to set inside the job.
109
+ interruptible: Whether the job should run on interruptible instances. They are cheaper but can be preempted.
110
+ image_credentials: The credentials used to pull the image. Required if the image is private.
111
+ This should be the name of the respective credentials secret created on the Lightning AI platform.
112
+ cloud_account_auth: Whether to authenticate with the cloud account to pull the image.
113
+ Required if the registry is part of a cloud provider (e.g. ECR).
114
+ artifacts_local: The path of inside the docker container, you want to persist images from.
115
+ CAUTION: When setting this to "/", it will effectively erase your container.
116
+ Only supported for jobs with a docker image compute environment.
117
+ artifacts_remote: The remote storage to persist your artifacts to.
118
+ Should be of format <CONNECTION_TYPE>:<CONNECTION_NAME>:<PATH_WITHIN_CONNECTION>.
119
+ PATH_WITHIN_CONNECTION hereby is a path relative to the connection's root.
120
+ E.g. efs:data:some-path would result in an EFS connection named `data` and to the path `some-path`
121
+ within it.
122
+ Note that the connection needs to be added to the teamspace already in order for it to be found.
123
+ Only supported for jobs with a docker image compute environment.
124
+ """
125
+ ret_val = super().run(
126
+ name=name,
127
+ num_machines=num_machines,
128
+ machine=machine,
129
+ command=command,
130
+ studio=studio,
131
+ image=image,
132
+ teamspace=teamspace,
133
+ org=org,
134
+ user=user,
135
+ cloud_account=cloud_account,
136
+ env=env,
137
+ interruptible=interruptible,
138
+ image_credentials=image_credentials,
139
+ cloud_account_auth=cloud_account_auth,
140
+ artifacts_local=artifacts_local,
141
+ artifacts_remote=artifacts_remote,
142
+ cluster=cluster, # deprecated in favor of cloud_account
143
+ )
144
+ # required for typing with "MMT"
145
+ assert isinstance(ret_val, cls)
146
+
147
+ msg = "Multi-Machine Job was successfully launched."
148
+
149
+ with suppress(NotImplementedError):
150
+ msg += f" View it at {ret_val.link}"
151
+
152
+ _logger.info(msg)
153
+ return ret_val
154
+
155
+ def _submit(
156
+ self,
157
+ num_machines: int,
158
+ machine: "Machine",
159
+ command: Optional[str] = None,
160
+ studio: Optional["Studio"] = None,
161
+ image: Optional[str] = None,
162
+ env: Optional[Dict[str, str]] = None,
163
+ interruptible: bool = False,
164
+ cloud_account: Optional[str] = None,
165
+ image_credentials: Optional[str] = None,
166
+ cloud_account_auth: bool = False,
167
+ artifacts_local: Optional[str] = None,
168
+ artifacts_remote: Optional[str] = None,
169
+ ) -> "MMT":
170
+ """Submit a new multi-machine job to the Lightning AI platform.
171
+
172
+ Args:
173
+ num_machines: The number of machines to run on.
174
+ machine: The machine type to run the job on. One of {", ".join(_MACHINE_VALUES)}.
175
+ command: The command to run inside your job. Required if using a studio. Optional if using an image.
176
+ If not provided for images, will run the container entrypoint and default command.
177
+ studio: The studio env to run the job with. Mutually exclusive with image.
178
+ image: The docker image to run the job with. Mutually exclusive with studio.
179
+ env: Environment variables to set inside the job.
180
+ interruptible: Whether the job should run on interruptible instances. They are cheaper but can be preempted.
181
+ cloud_account: The cloud account to run the job on.
182
+ Defaults to the studio cloud account if running with studio compute env.
183
+ If not provided will fall back to the teamspaces default cloud account.
184
+ image_credentials: The credentials used to pull the image. Required if the image is private.
185
+ This should be the name of the respective credentials secret created on the Lightning AI platform.
186
+ cloud_account_auth: Whether to authenticate with the cloud account to pull the image.
187
+ Required if the registry is part of a cloud provider (e.g. ECR).
188
+ artifacts_local: The path of inside the docker container, you want to persist images from.
189
+ CAUTION: When setting this to "/", it will effectively erase your container.
190
+ Only supported for jobs with a docker image compute environment.
191
+ artifacts_remote: The remote storage to persist your artifacts to.
192
+ Should be of format <CONNECTION_TYPE>:<CONNECTION_NAME>:<PATH_WITHIN_CONNECTION>.
193
+ PATH_WITHIN_CONNECTION hereby is a path relative to the connection's root.
194
+ E.g. efs:data:some-path would result in an EFS connection named `data` and to the path `some-path`
195
+ within it.
196
+ Note that the connection needs to be added to the teamspace already in order for it to be found.
197
+ Only supported for jobs with a docker image compute environment.
198
+ """
199
+ self._job = self._internal_mmt._submit(
200
+ num_machines=num_machines,
201
+ machine=machine,
202
+ cloud_account=cloud_account,
203
+ command=command,
204
+ studio=studio,
205
+ image=image,
206
+ env=env,
207
+ interruptible=interruptible,
208
+ image_credentials=image_credentials,
209
+ cloud_account_auth=cloud_account_auth,
210
+ artifacts_local=artifacts_local,
211
+ artifacts_remote=artifacts_remote,
212
+ )
213
+ return self
214
+
215
+ def stop(self) -> None:
216
+ """Stops the job."""
217
+ return self._internal_mmt.stop()
218
+
219
+ def delete(self) -> None:
220
+ """Deletes the job.
221
+
222
+ Caution: This also deletes all artifacts and snapshots associated with the job.
223
+ """
224
+ return self._internal_mmt.delete()
225
+
226
+ @property
227
+ def status(self) -> "Status":
228
+ """The current status of the job (accumulated over all machines)."""
229
+ return self._internal_mmt.status
230
+
231
+ @property
232
+ def machines(self) -> Tuple[MMTMachine, ...]:
233
+ """Returns the sub-jobs for each individual instance."""
234
+ return self._internal_mmt.machines
235
+
236
+ @property
237
+ def machine(self) -> "Machine":
238
+ """Returns the machine type this job is running on."""
239
+ return self._internal_mmt.machine
240
+
241
+ @property
242
+ def artifact_path(self) -> Optional[str]:
243
+ """Path to the artifacts created by the job within the distributed teamspace filesystem."""
244
+ return self._internal_mmt.artifact_path
245
+
246
+ @property
247
+ def snapshot_path(self) -> Optional[str]:
248
+ """Path to the studio snapshot used to create the job within the distributed teamspace filesystem."""
249
+ return self._internal_mmt.snapshot_path
250
+
251
+ @property
252
+ def share_path(self) -> Optional[str]:
253
+ """Path to the jobs share path."""
254
+ return None
255
+
256
+ def _update_internal_job(self) -> None:
257
+ return self._internal_mmt._update_internal_job()
258
+
259
+ @property
260
+ def name(self) -> str:
261
+ """The job's name."""
262
+ return self._internal_mmt.name
263
+
264
+ @property
265
+ def teamspace(self) -> "Teamspace":
266
+ """The teamspace the job is part of."""
267
+ return self._internal_mmt._teamspace
268
+
269
+ @property
270
+ def link(self) -> str:
271
+ return self._internal_mmt.link
272
+
273
+ def __getattr__(self, key: str) -> Any:
274
+ """Forward the attribute lookup to the internal job implementation."""
275
+ try:
276
+ return getattr(super(), key)
277
+ except AttributeError:
278
+ return getattr(self._internal_mmt, key)
279
+
280
+ @property
281
+ def _guaranteed_job(self) -> Any:
282
+ return self._internal_mmt._guaranteed_job
@@ -0,0 +1,185 @@
1
+ from typing import TYPE_CHECKING, Dict, Optional, Tuple, Union
2
+
3
+ from lightning_sdk.api.mmt_api import MMTApiV1
4
+ from lightning_sdk.job.v1 import _internal_status_to_external_status
5
+ from lightning_sdk.job.work import Work
6
+
7
+ if TYPE_CHECKING:
8
+ from lightning_sdk.machine import Machine
9
+ from lightning_sdk.organization import Organization
10
+ from lightning_sdk.status import Status
11
+ from lightning_sdk.studio import Studio
12
+ from lightning_sdk.teamspace import Teamspace
13
+ from lightning_sdk.user import User
14
+
15
+ from lightning_sdk.mmt.base import _BaseMMT
16
+
17
+
18
+ class _MMTV1(_BaseMMT):
19
+ """V1 Implementation of Multi-Machine Training."""
20
+
21
+ def __init__(
22
+ self,
23
+ name: str,
24
+ teamspace: Union[str, "Teamspace", None] = None,
25
+ org: Union[str, "Organization", None] = None,
26
+ user: Union[str, "User", None] = None,
27
+ *,
28
+ _fetch_job: bool = True,
29
+ ) -> None:
30
+ """Fetch already existing jobs.
31
+
32
+ Args:
33
+ name: the name of the job
34
+ teamspace: the teamspace the job is part of
35
+ org: the name of the organization owning the :param`teamspace` in case it is owned by an org
36
+ user: the name of the user owning the :param`teamspace`
37
+ in case it is owned directly by a user instead of an org.
38
+ """
39
+ self._job_api = MMTApiV1()
40
+ super().__init__(name=name, teamspace=teamspace, org=org, user=user, _fetch_job=_fetch_job)
41
+
42
+ def _submit(
43
+ self,
44
+ num_machines: int,
45
+ machine: "Machine",
46
+ command: Optional[str] = None,
47
+ studio: Optional["Studio"] = None,
48
+ image: Optional[str] = None,
49
+ env: Optional[Dict[str, str]] = None,
50
+ interruptible: bool = False,
51
+ cloud_account: Optional[str] = None,
52
+ image_credentials: Optional[str] = None,
53
+ cloud_account_auth: bool = False,
54
+ artifacts_local: Optional[str] = None,
55
+ artifacts_remote: Optional[str] = None,
56
+ ) -> "_MMTV1":
57
+ """Submit a new multi-machine job to the Lightning AI platform.
58
+
59
+ Args:
60
+ num_machines: The number of machines to run on.
61
+ machine: The machine type to run the job on. One of {", ".join(_MACHINE_VALUES)}.
62
+ command: The command to run inside your job. Required if using a studio. Optional if using an image.
63
+ If not provided for images, will run the container entrypoint and default command.
64
+ studio: The studio env to run the job with. Mutually exclusive with image.
65
+ image: The docker image to run the job with. Mutually exclusive with studio.
66
+ env: Environment variables to set inside the job.
67
+ interruptible: Whether the job should run on interruptible instances. They are cheaper but can be preempted.
68
+ cloud_account: The cloud account to run the job on.
69
+ Defaults to the studio cloud account if running with studio compute env.
70
+ If not provided will fall back to the teamspaces default cloud account.
71
+ image_credentials: The credentials used to pull the image. Required if the image is private.
72
+ This should be the name of the respective credentials secret created on the Lightning AI platform.
73
+ cloud_account_auth: Whether to authenticate with the cloud account to pull the image.
74
+ Required if the registry is part of a cloud provider (e.g. ECR).
75
+ artifacts_local: The path of inside the docker container, you want to persist images from.
76
+ CAUTION: When setting this to "/", it will effectively erase your container.
77
+ Only supported for jobs with a docker image compute environment.
78
+ artifacts_remote: The remote storage to persist your artifacts to.
79
+ Should be of format <CONNECTION_TYPE>:<CONNECTION_NAME>:<PATH_WITHIN_CONNECTION>.
80
+ PATH_WITHIN_CONNECTION hereby is a path relative to the connection's root.
81
+ E.g. efs:data:some-path would result in an EFS connection named `data` and to the path `some-path`
82
+ within it.
83
+ Note that the connection needs to be added to the teamspace already in order for it to be found.
84
+ Only supported for jobs with a docker image compute environment.
85
+ """
86
+ if studio is None:
87
+ raise ValueError("Studio is required for submitting jobs")
88
+ if image is not None or image_credentials is not None or cloud_account_auth:
89
+ raise ValueError("Image is not supported for submitting jobs")
90
+
91
+ if artifacts_local is not None or artifacts_remote is not None:
92
+ raise ValueError("Specifying how to persist artifacts is not yet supported with jobs")
93
+
94
+ if env is not None:
95
+ raise ValueError("Environment variables are not supported for submitting jobs")
96
+ if command is None:
97
+ raise ValueError("Command is required for submitting multi-machine jobs")
98
+
99
+ _submitted = self._job_api.submit_job(
100
+ name=self._name,
101
+ num_machines=num_machines,
102
+ command=command,
103
+ studio_id=studio._studio.id,
104
+ teamspace_id=self._teamspace.id,
105
+ cloud_account=cloud_account or "",
106
+ machine=machine,
107
+ interruptible=interruptible,
108
+ strategy="parallel",
109
+ )
110
+
111
+ self._name = _submitted.name
112
+ self._job = _submitted
113
+ return self
114
+
115
+ def _update_internal_job(self) -> None:
116
+ try:
117
+ self._job = self._job_api.get_job(self._name, self.teamspace.id)
118
+ except ValueError as e:
119
+ raise ValueError(f"Job {self._name} does not exist in Teamspace {self.teamspace.name}") from e
120
+
121
+ @property
122
+ def machines(self) -> Tuple["Work", ...]:
123
+ """Returns the sub-jobs for each individual instance."""
124
+ works = self._job_api.list_works(self._guaranteed_job.id, self.teamspace.id)
125
+
126
+ return tuple(Work(w.id, self, self.teamspace) for w in works)
127
+
128
+ def stop(self) -> None:
129
+ """Stops the job."""
130
+ self._job_api.stop_job(self._guaranteed_job.id, self.teamspace.id)
131
+
132
+ def delete(self) -> None:
133
+ """Deletes the job.
134
+
135
+ Caution: This also deletes all artifacts and snapshots associated with the job.
136
+ """
137
+ self._job_api.delete_job(self._guaranteed_job.id, self.teamspace.id)
138
+
139
+ @property
140
+ def status(self) -> "Status":
141
+ """The current status of the job."""
142
+ try:
143
+ status = self._job_api.get_job_status(self._job.id, self.teamspace.id)
144
+ return _internal_status_to_external_status(status)
145
+ except Exception:
146
+ raise RuntimeError(
147
+ f"MMT {self._name} does not exist in Teamspace {self.teamspace.name}. Did you delete it?"
148
+ ) from None
149
+
150
+ @property
151
+ def artifact_path(self) -> Optional[str]:
152
+ """Path to the artifacts created by the job within the distributed teamspace filesystem."""
153
+ return f"/teamspace/jobs/{self.name}"
154
+
155
+ @property
156
+ def snapshot_path(self) -> Optional[str]:
157
+ """Path to the studio snapshot used to create the job within the distributed teamspace filesystem."""
158
+ return f"/teamspace/jobs/{self.name}/snapshot"
159
+
160
+ @property
161
+ def machine(self) -> "Machine":
162
+ """Returns the machine type this job is running on."""
163
+ return self.machines[0].machine
164
+
165
+ @property
166
+ def name(self) -> str:
167
+ """The job's name."""
168
+ return self._name
169
+
170
+ @property
171
+ def teamspace(self) -> "Teamspace":
172
+ """The teamspace the job is part of."""
173
+ return self._teamspace
174
+
175
+ @property
176
+ def link(self) -> str:
177
+ return f"https://lightning.ai/{self.teamspace.owner.name}/{self.teamspace.name}/studios/{self._job_api.get_studio_name(self._guaranteed_job)}/app?app_id=mmt&app_tab=Runs&job_name={self.name}"
178
+
179
+ # the following and functions are solely to make the Work class function
180
+ @property
181
+ def _id(self) -> str:
182
+ return self._guaranteed_job.id
183
+
184
+ def _name_filter(self, name: str) -> str:
185
+ return name.replace("root.", "")
@@ -0,0 +1,193 @@
1
+ from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union
2
+
3
+ from lightning_sdk.api.mmt_api import MMTApiV2
4
+
5
+ if TYPE_CHECKING:
6
+ from lightning_sdk.job.job import Job
7
+ from lightning_sdk.machine import Machine
8
+ from lightning_sdk.organization import Organization
9
+ from lightning_sdk.status import Status
10
+ from lightning_sdk.studio import Studio
11
+ from lightning_sdk.teamspace import Teamspace
12
+ from lightning_sdk.user import User
13
+
14
+ from lightning_sdk.mmt.base import _BaseMMT
15
+
16
+
17
+ class _MMTV2(_BaseMMT):
18
+ """New implementation of Multi-Machine Training."""
19
+
20
+ def __init__(
21
+ self,
22
+ name: str,
23
+ teamspace: Union[str, "Teamspace", None] = None,
24
+ org: Union[str, "Organization", None] = None,
25
+ user: Union[str, "User", None] = None,
26
+ *,
27
+ _fetch_job: bool = True,
28
+ ) -> None:
29
+ """Fetch already existing jobs.
30
+
31
+ Args:
32
+ name: the name of the job
33
+ teamspace: the teamspace the job is part of
34
+ org: the name of the organization owning the :param`teamspace` in case it is owned by an org
35
+ user: the name of the user owning the :param`teamspace`
36
+ in case it is owned directly by a user instead of an org.
37
+ """
38
+ self._job_api = MMTApiV2()
39
+ super().__init__(name=name, teamspace=teamspace, org=org, user=user, _fetch_job=_fetch_job)
40
+
41
+ def _submit(
42
+ self,
43
+ num_machines: int,
44
+ machine: "Machine",
45
+ command: Optional[str] = None,
46
+ studio: Optional["Studio"] = None,
47
+ image: Optional[str] = None,
48
+ env: Optional[Dict[str, str]] = None,
49
+ interruptible: bool = False,
50
+ cloud_account: Optional[str] = None,
51
+ image_credentials: Optional[str] = None,
52
+ cloud_account_auth: bool = False,
53
+ artifacts_local: Optional[str] = None,
54
+ artifacts_remote: Optional[str] = None,
55
+ ) -> "_MMTV2":
56
+ """Submit a new multi-machine job to the Lightning AI platform.
57
+
58
+ Args:
59
+ num_machines: The number of machines to run on.
60
+ machine: The machine type to run the job on. One of {", ".join(_MACHINE_VALUES)}.
61
+ command: The command to run inside your job. Required if using a studio. Optional if using an image.
62
+ If not provided for images, will run the container entrypoint and default command.
63
+ studio: The studio env to run the job with. Mutually exclusive with image.
64
+ image: The docker image to run the job with. Mutually exclusive with studio.
65
+ env: Environment variables to set inside the job.
66
+ interruptible: Whether the job should run on interruptible instances. They are cheaper but can be preempted.
67
+ cloud_account: The cloud account to run the job on.
68
+ Defaults to the studio cloud account if running with studio compute env.
69
+ If not provided will fall back to the teamspaces default cloud account.
70
+ image_credentials: The credentials used to pull the image. Required if the image is private.
71
+ This should be the name of the respective credentials secret created on the Lightning AI platform.
72
+ cloud_account_auth: Whether to authenticate with the cloud account to pull the image.
73
+ Required if the registry is part of a cloud provider (e.g. ECR).
74
+ artifacts_local: The path of inside the docker container, you want to persist images from.
75
+ CAUTION: When setting this to "/", it will effectively erase your container.
76
+ Only supported for jobs with a docker image compute environment.
77
+ artifacts_remote: The remote storage to persist your artifacts to.
78
+ Should be of format <CONNECTION_TYPE>:<CONNECTION_NAME>:<PATH_WITHIN_CONNECTION>.
79
+ PATH_WITHIN_CONNECTION hereby is a path relative to the connection's root.
80
+ E.g. efs:data:some-path would result in an EFS connection named `data` and to the path `some-path`
81
+ within it.
82
+ Note that the connection needs to be added to the teamspace already in order for it to be found.
83
+ Only supported for jobs with a docker image compute environment.
84
+ """
85
+ # Command is required if Studio is provided to know what to run
86
+ # Image is mutually exclusive with Studio
87
+ # Command is optional for Image
88
+ # Either image or studio must be provided
89
+ if studio is not None:
90
+ studio_id = studio._studio.id
91
+ if image is not None:
92
+ raise ValueError(
93
+ "image and studio are mutually exclusive as both define the environment to run the job in"
94
+ )
95
+ if command is None:
96
+ raise ValueError("command is required when using a studio")
97
+ else:
98
+ studio_id = None
99
+ if image is None:
100
+ raise ValueError("either image or studio must be provided")
101
+ submitted = self._job_api.submit_job(
102
+ name=self.name,
103
+ num_machines=num_machines,
104
+ command=command,
105
+ cloud_account=cloud_account,
106
+ teamspace_id=self._teamspace.id,
107
+ studio_id=studio_id,
108
+ image=image,
109
+ machine=machine,
110
+ interruptible=interruptible,
111
+ env=env,
112
+ image_credentials=image_credentials,
113
+ cloud_account_auth=cloud_account_auth,
114
+ artifacts_local=artifacts_local,
115
+ artifacts_remote=artifacts_remote,
116
+ )
117
+ self._job = submitted
118
+ self._name = submitted.name
119
+ return self
120
+
121
+ @property
122
+ def machines(self) -> Tuple["Job", ...]:
123
+ """Returns the sub-jobs for each individual instance."""
124
+ from lightning_sdk.job import Job
125
+
126
+ return tuple(
127
+ Job(name=j.name, teamspace=self.teamspace)
128
+ for j in self._job_api.list_mmt_subjobs(self._guaranteed_job.id, self.teamspace.id)
129
+ )
130
+
131
+ def stop(self) -> None:
132
+ """Stops the job."""
133
+ self._job_api.stop_job(job_id=self._guaranteed_job.id, teamspace_id=self._teamspace.id)
134
+
135
+ def delete(self) -> None:
136
+ """Deletes the job.
137
+
138
+ Caution: This also deletes all artifacts and snapshots associated with the job.
139
+ """
140
+ self._job_api.delete_job(
141
+ job_id=self._guaranteed_job.id,
142
+ teamspace_id=self._teamspace.id,
143
+ )
144
+
145
+ @property
146
+ def _latest_job(self) -> Any:
147
+ """Guarantees to fetch the latest version of a job before returning it."""
148
+ self._update_internal_job()
149
+ return self._job
150
+
151
+ @property
152
+ def status(self) -> "Status":
153
+ """The current status of the job."""
154
+ return self._job_api._job_state_to_external(self._latest_job.state)
155
+
156
+ @property
157
+ def artifact_path(self) -> Optional[str]:
158
+ """Path to the artifacts created by the job within the distributed teamspace filesystem."""
159
+ # TODO: Since grouping for those is not done yet on the BE, we cannot yet have a unified link here
160
+ raise NotImplementedError
161
+
162
+ @property
163
+ def snapshot_path(self) -> Optional[str]:
164
+ """Path to the studio snapshot used to create the job within the distributed teamspace filesystem."""
165
+ # TODO: Since grouping for those is not done yet on the BE, we cannot yet have a unified link here
166
+ raise NotImplementedError
167
+
168
+ @property
169
+ def machine(self) -> "Machine":
170
+ """Returns the machine type this job is running on."""
171
+ return self._job_api._get_job_machine_from_spec(self._guaranteed_job.spec)
172
+
173
+ def _update_internal_job(self) -> None:
174
+ if getattr(self, "_job", None) is None:
175
+ self._job = self._job_api.get_job_by_name(name=self._name, teamspace_id=self._teamspace.id)
176
+ return
177
+
178
+ self._job = self._job_api.get_job(job_id=self._job.id, teamspace_id=self._teamspace.id)
179
+
180
+ @property
181
+ def name(self) -> str:
182
+ """The job's name."""
183
+ return self._name
184
+
185
+ @property
186
+ def teamspace(self) -> "Teamspace":
187
+ """The teamspace the job is part of."""
188
+ return self._teamspace
189
+
190
+ @property
191
+ def link(self) -> str:
192
+ # TODO: Since we don't have a UI for this yet, we can't have a link
193
+ raise NotImplementedError