dstack 0.18.41__py3-none-any.whl → 0.18.43__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. dstack/_internal/cli/commands/__init__.py +2 -1
  2. dstack/_internal/cli/commands/apply.py +4 -2
  3. dstack/_internal/cli/commands/attach.py +21 -1
  4. dstack/_internal/cli/commands/completion.py +20 -0
  5. dstack/_internal/cli/commands/delete.py +3 -1
  6. dstack/_internal/cli/commands/fleet.py +2 -1
  7. dstack/_internal/cli/commands/gateway.py +7 -2
  8. dstack/_internal/cli/commands/logs.py +3 -2
  9. dstack/_internal/cli/commands/stats.py +2 -1
  10. dstack/_internal/cli/commands/stop.py +2 -1
  11. dstack/_internal/cli/commands/volume.py +2 -1
  12. dstack/_internal/cli/main.py +6 -0
  13. dstack/_internal/cli/services/completion.py +86 -0
  14. dstack/_internal/cli/services/configurators/run.py +10 -17
  15. dstack/_internal/cli/utils/fleet.py +5 -1
  16. dstack/_internal/cli/utils/volume.py +9 -0
  17. dstack/_internal/core/backends/aws/compute.py +24 -11
  18. dstack/_internal/core/backends/aws/resources.py +3 -3
  19. dstack/_internal/core/backends/azure/compute.py +14 -8
  20. dstack/_internal/core/backends/azure/resources.py +2 -0
  21. dstack/_internal/core/backends/base/compute.py +102 -2
  22. dstack/_internal/core/backends/base/offers.py +7 -1
  23. dstack/_internal/core/backends/cudo/compute.py +8 -4
  24. dstack/_internal/core/backends/datacrunch/compute.py +10 -4
  25. dstack/_internal/core/backends/gcp/auth.py +19 -13
  26. dstack/_internal/core/backends/gcp/compute.py +27 -20
  27. dstack/_internal/core/backends/gcp/resources.py +3 -10
  28. dstack/_internal/core/backends/kubernetes/compute.py +4 -3
  29. dstack/_internal/core/backends/lambdalabs/compute.py +9 -3
  30. dstack/_internal/core/backends/nebius/compute.py +2 -2
  31. dstack/_internal/core/backends/oci/compute.py +10 -4
  32. dstack/_internal/core/backends/runpod/compute.py +11 -4
  33. dstack/_internal/core/backends/tensordock/compute.py +14 -3
  34. dstack/_internal/core/backends/vastai/compute.py +12 -2
  35. dstack/_internal/core/backends/vultr/api_client.py +3 -3
  36. dstack/_internal/core/backends/vultr/compute.py +9 -3
  37. dstack/_internal/core/models/backends/aws.py +2 -0
  38. dstack/_internal/core/models/backends/base.py +1 -0
  39. dstack/_internal/core/models/configurations.py +0 -1
  40. dstack/_internal/core/models/runs.py +3 -3
  41. dstack/_internal/core/models/volumes.py +23 -0
  42. dstack/_internal/core/services/__init__.py +5 -1
  43. dstack/_internal/core/services/configs/__init__.py +3 -0
  44. dstack/_internal/server/background/tasks/common.py +22 -0
  45. dstack/_internal/server/background/tasks/process_instances.py +13 -21
  46. dstack/_internal/server/background/tasks/process_running_jobs.py +13 -16
  47. dstack/_internal/server/background/tasks/process_submitted_jobs.py +12 -7
  48. dstack/_internal/server/background/tasks/process_terminating_jobs.py +7 -2
  49. dstack/_internal/server/background/tasks/process_volumes.py +11 -1
  50. dstack/_internal/server/migrations/versions/a751ef183f27_move_attachment_data_to_volumes_.py +34 -0
  51. dstack/_internal/server/models.py +17 -19
  52. dstack/_internal/server/routers/logs.py +3 -0
  53. dstack/_internal/server/services/backends/configurators/aws.py +31 -1
  54. dstack/_internal/server/services/backends/configurators/gcp.py +8 -15
  55. dstack/_internal/server/services/config.py +11 -1
  56. dstack/_internal/server/services/fleets.py +5 -1
  57. dstack/_internal/server/services/jobs/__init__.py +14 -11
  58. dstack/_internal/server/services/jobs/configurators/dev.py +1 -3
  59. dstack/_internal/server/services/jobs/configurators/task.py +1 -3
  60. dstack/_internal/server/services/logs/__init__.py +78 -0
  61. dstack/_internal/server/services/{logs.py → logs/aws.py} +12 -207
  62. dstack/_internal/server/services/logs/base.py +47 -0
  63. dstack/_internal/server/services/logs/filelog.py +110 -0
  64. dstack/_internal/server/services/logs/gcp.py +165 -0
  65. dstack/_internal/server/services/offers.py +7 -7
  66. dstack/_internal/server/services/pools.py +19 -20
  67. dstack/_internal/server/services/proxy/routers/service_proxy.py +14 -7
  68. dstack/_internal/server/services/runner/client.py +8 -5
  69. dstack/_internal/server/services/volumes.py +68 -9
  70. dstack/_internal/server/settings.py +3 -0
  71. dstack/_internal/server/statics/index.html +1 -1
  72. dstack/_internal/server/statics/{main-ad5150a441de98cd8987.css → main-7510e71dfa9749a4e70e.css} +1 -1
  73. dstack/_internal/server/statics/{main-2ac66bfcbd2e39830b88.js → main-fe8fd9db55df8d10e648.js} +66 -66
  74. dstack/_internal/server/statics/{main-2ac66bfcbd2e39830b88.js.map → main-fe8fd9db55df8d10e648.js.map} +1 -1
  75. dstack/_internal/server/testing/common.py +46 -17
  76. dstack/api/_public/runs.py +1 -1
  77. dstack/version.py +2 -2
  78. {dstack-0.18.41.dist-info → dstack-0.18.43.dist-info}/METADATA +4 -3
  79. {dstack-0.18.41.dist-info → dstack-0.18.43.dist-info}/RECORD +97 -86
  80. tests/_internal/core/backends/base/__init__.py +0 -0
  81. tests/_internal/core/backends/base/test_compute.py +56 -0
  82. tests/_internal/server/background/tasks/test_process_running_jobs.py +2 -1
  83. tests/_internal/server/background/tasks/test_process_submitted_jobs.py +5 -3
  84. tests/_internal/server/background/tasks/test_process_terminating_jobs.py +11 -6
  85. tests/_internal/server/conftest.py +4 -5
  86. tests/_internal/server/routers/test_backends.py +1 -0
  87. tests/_internal/server/routers/test_logs.py +1 -1
  88. tests/_internal/server/routers/test_runs.py +2 -2
  89. tests/_internal/server/routers/test_volumes.py +9 -2
  90. tests/_internal/server/services/runner/test_client.py +22 -3
  91. tests/_internal/server/services/test_logs.py +3 -3
  92. tests/_internal/server/services/test_offers.py +167 -0
  93. tests/_internal/server/services/test_pools.py +105 -1
  94. {dstack-0.18.41.dist-info → dstack-0.18.43.dist-info}/LICENSE.md +0 -0
  95. {dstack-0.18.41.dist-info → dstack-0.18.43.dist-info}/WHEEL +0 -0
  96. {dstack-0.18.41.dist-info → dstack-0.18.43.dist-info}/entry_points.txt +0 -0
  97. {dstack-0.18.41.dist-info → dstack-0.18.43.dist-info}/top_level.txt +0 -0
@@ -462,19 +462,19 @@ def filter_pool_instances(
462
462
  zones = [z for z in zones if z in volume_zones]
463
463
 
464
464
  if multinode:
465
- if not backend_types:
465
+ if backend_types is None:
466
466
  backend_types = BACKENDS_WITH_MULTINODE_SUPPORT
467
467
  backend_types = [b for b in backend_types if b in BACKENDS_WITH_MULTINODE_SUPPORT]
468
468
 
469
469
  # For multi-node, restrict backend and region.
470
470
  # The default behavior is to provision all nodes in the same backend and region.
471
471
  if master_job_provisioning_data is not None:
472
- if not backend_types:
472
+ if backend_types is None:
473
473
  backend_types = [master_job_provisioning_data.get_base_backend()]
474
474
  backend_types = [
475
475
  b for b in backend_types if b == master_job_provisioning_data.get_base_backend()
476
476
  ]
477
- if not regions:
477
+ if regions is None:
478
478
  regions = [master_job_provisioning_data.region]
479
479
  regions = [r for r in regions if r == master_job_provisioning_data.region]
480
480
 
@@ -487,24 +487,23 @@ def filter_pool_instances(
487
487
  continue
488
488
  if status is not None and instance.status != status:
489
489
  continue
490
- if backend_types is not None and instance.backend not in backend_types:
491
- continue
492
- if regions is not None and instance.region not in regions:
493
- continue
494
490
  jpd = get_instance_provisioning_data(instance)
495
- if (
496
- jpd is not None
497
- and profile.instance_types is not None
498
- and jpd.instance_type.name not in profile.instance_types
499
- ):
500
- continue
501
- if (
502
- jpd is not None
503
- and jpd.availability_zone is not None
504
- and zones is not None
505
- and jpd.availability_zone not in zones
506
- ):
507
- continue
491
+ if jpd is not None:
492
+ if backend_types is not None and jpd.get_base_backend() not in backend_types:
493
+ continue
494
+ if regions is not None and jpd.region not in regions:
495
+ continue
496
+ if (
497
+ profile.instance_types is not None
498
+ and jpd.instance_type.name not in profile.instance_types
499
+ ):
500
+ continue
501
+ if (
502
+ jpd.availability_zone is not None
503
+ and zones is not None
504
+ and jpd.availability_zone not in zones
505
+ ):
506
+ continue
508
507
  if instance.total_blocks is None:
509
508
  # Still provisioning, we don't know yet if it shared or not
510
509
  continue
@@ -13,21 +13,28 @@ from dstack._internal.proxy.lib.repo import BaseProxyRepo
13
13
  from dstack._internal.proxy.lib.services.service_connection import ServiceConnectionPool
14
14
  from dstack._internal.server.services.proxy.services import service_proxy
15
15
 
16
- REDIRECTED_HTTP_METHODS = ["GET", "POST", "PUT", "DELETE", "PATCH", "HEAD"]
17
- PROXIED_HTTP_METHODS = REDIRECTED_HTTP_METHODS + ["OPTIONS"]
18
-
19
-
20
16
  router = APIRouter()
21
17
 
22
18
 
23
- @router.api_route("/{project_name}/{run_name}", methods=REDIRECTED_HTTP_METHODS)
24
- async def redirect_to_service_root(request: Request) -> Response:
19
+ @router.get("/{project_name}/{run_name}")
20
+ @router.post("/{project_name}/{run_name}")
21
+ @router.put("/{project_name}/{run_name}")
22
+ @router.delete("/{project_name}/{run_name}")
23
+ @router.patch("/{project_name}/{run_name}")
24
+ @router.head("/{project_name}/{run_name}")
25
+ async def redirect_to_service_root(request: Request, project_name: str, run_name: str) -> Response:
25
26
  url = URL(str(request.url))
26
27
  url = url.replace(path=url.path + "/")
27
28
  return RedirectResponse(url, status.HTTP_308_PERMANENT_REDIRECT)
28
29
 
29
30
 
30
- @router.api_route("/{project_name}/{run_name}/{path:path}", methods=PROXIED_HTTP_METHODS)
31
+ @router.get("/{project_name}/{run_name}/{path:path}")
32
+ @router.post("/{project_name}/{run_name}/{path:path}")
33
+ @router.put("/{project_name}/{run_name}/{path:path}")
34
+ @router.delete("/{project_name}/{run_name}/{path:path}")
35
+ @router.patch("/{project_name}/{run_name}/{path:path}")
36
+ @router.head("/{project_name}/{run_name}/{path:path}")
37
+ @router.options("/{project_name}/{run_name}/{path:path}")
31
38
  async def service_reverse_proxy(
32
39
  project_name: str,
33
40
  run_name: str,
@@ -239,6 +239,7 @@ class ShimClient:
239
239
  host_ssh_user: str,
240
240
  host_ssh_keys: list[str],
241
241
  container_ssh_keys: list[str],
242
+ instance_id: str,
242
243
  ) -> None:
243
244
  if not self.is_api_v2_supported():
244
245
  raise ShimAPIVersionError()
@@ -255,7 +256,7 @@ class ShimClient:
255
256
  memory=_memory_to_bytes(memory), # None = 0 = "all available"
256
257
  shm_size=_memory_to_bytes(shm_size), # None = 0 = "use default value"
257
258
  network_mode=network_mode,
258
- volumes=[_volume_to_shim_volume_info(v) for v in volumes],
259
+ volumes=[_volume_to_shim_volume_info(v, instance_id) for v in volumes],
259
260
  volume_mounts=volume_mounts,
260
261
  instance_mounts=instance_mounts,
261
262
  host_ssh_user=host_ssh_user,
@@ -303,6 +304,7 @@ class ShimClient:
303
304
  mounts: List[VolumeMountPoint],
304
305
  volumes: List[Volume],
305
306
  instance_mounts: List[InstanceMountPoint],
307
+ instance_id: str,
306
308
  ) -> bool:
307
309
  """
308
310
  Returns `True` if submitted and `False` if the shim already has a job (`409 Conflict`).
@@ -320,7 +322,7 @@ class ShimClient:
320
322
  ssh_user=ssh_user,
321
323
  ssh_key=ssh_key,
322
324
  mounts=mounts,
323
- volumes=[_volume_to_shim_volume_info(v) for v in volumes],
325
+ volumes=[_volume_to_shim_volume_info(v, instance_id) for v in volumes],
324
326
  instance_mounts=instance_mounts,
325
327
  )
326
328
  resp = self._request("POST", "/api/submit", body)
@@ -398,10 +400,11 @@ def health_response_to_health_status(data: HealthcheckResponse) -> HealthStatus:
398
400
  )
399
401
 
400
402
 
401
- def _volume_to_shim_volume_info(volume: Volume) -> ShimVolumeInfo:
403
+ def _volume_to_shim_volume_info(volume: Volume, instance_id: str) -> ShimVolumeInfo:
402
404
  device_name = None
403
- if volume.attachment_data is not None:
404
- device_name = volume.attachment_data.device_name
405
+ attachment_data = volume.get_attachment_data_for_instance(instance_id)
406
+ if attachment_data is not None:
407
+ device_name = attachment_data.device_name
405
408
  return ShimVolumeInfo(
406
409
  backend=volume.configuration.backend.value,
407
410
  name=volume.name,
@@ -15,19 +15,28 @@ from dstack._internal.core.errors import (
15
15
  from dstack._internal.core.models.users import GlobalRole
16
16
  from dstack._internal.core.models.volumes import (
17
17
  Volume,
18
+ VolumeAttachment,
18
19
  VolumeAttachmentData,
19
20
  VolumeConfiguration,
21
+ VolumeInstance,
20
22
  VolumeProvisioningData,
21
23
  VolumeStatus,
22
24
  )
23
25
  from dstack._internal.core.services import validate_dstack_resource_name
24
26
  from dstack._internal.server.db import get_db
25
- from dstack._internal.server.models import ProjectModel, UserModel, VolumeModel
27
+ from dstack._internal.server.models import (
28
+ InstanceModel,
29
+ ProjectModel,
30
+ UserModel,
31
+ VolumeAttachmentModel,
32
+ VolumeModel,
33
+ )
26
34
  from dstack._internal.server.services import backends as backends_services
27
35
  from dstack._internal.server.services.locking import (
28
36
  get_locker,
29
37
  string_to_lock_id,
30
38
  )
39
+ from dstack._internal.server.services.pools import get_instance_provisioning_data
31
40
  from dstack._internal.server.services.projects import list_project_models, list_user_project_models
32
41
  from dstack._internal.utils import common, random_names
33
42
  from dstack._internal.utils.logging import get_logger
@@ -106,8 +115,13 @@ async def list_projects_volume_models(
106
115
  .order_by(*order_by)
107
116
  .limit(limit)
108
117
  .options(joinedload(VolumeModel.user))
118
+ .options(
119
+ joinedload(VolumeModel.attachments)
120
+ .joinedload(VolumeAttachmentModel.instance)
121
+ .joinedload(InstanceModel.fleet)
122
+ )
109
123
  )
110
- volume_models = list(res.scalars().all())
124
+ volume_models = list(res.unique().scalars().all())
111
125
  return volume_models
112
126
 
113
127
 
@@ -134,9 +148,16 @@ async def list_project_volume_models(
134
148
  if not include_deleted:
135
149
  filters.append(VolumeModel.deleted == False)
136
150
  res = await session.execute(
137
- select(VolumeModel).where(*filters).options(joinedload(VolumeModel.user))
151
+ select(VolumeModel)
152
+ .where(*filters)
153
+ .options(joinedload(VolumeModel.user))
154
+ .options(
155
+ joinedload(VolumeModel.attachments)
156
+ .joinedload(VolumeAttachmentModel.instance)
157
+ .joinedload(InstanceModel.fleet)
158
+ )
138
159
  )
139
- return list(res.scalars().all())
160
+ return list(res.unique().scalars().all())
140
161
 
141
162
 
142
163
  async def get_volume_by_name(
@@ -163,9 +184,16 @@ async def get_project_volume_model_by_name(
163
184
  if not include_deleted:
164
185
  filters.append(VolumeModel.deleted == False)
165
186
  res = await session.execute(
166
- select(VolumeModel).where(*filters).options(joinedload(VolumeModel.user))
187
+ select(VolumeModel)
188
+ .where(*filters)
189
+ .options(joinedload(VolumeModel.user))
190
+ .options(
191
+ joinedload(VolumeModel.attachments)
192
+ .joinedload(VolumeAttachmentModel.instance)
193
+ .joinedload(InstanceModel.fleet)
194
+ )
167
195
  )
168
- return res.scalar_one_or_none()
196
+ return res.unique().scalar_one_or_none()
169
197
 
170
198
 
171
199
  async def create_volume(
@@ -205,10 +233,10 @@ async def create_volume(
205
233
  project=project,
206
234
  status=VolumeStatus.SUBMITTED,
207
235
  configuration=configuration.json(),
236
+ attachments=[],
208
237
  )
209
238
  session.add(volume_model)
210
239
  await session.commit()
211
- await session.refresh(volume_model)
212
240
  return volume_model_to_volume(volume_model)
213
241
 
214
242
 
@@ -234,13 +262,13 @@ async def delete_volumes(session: AsyncSession, project: ProjectModel, names: Li
234
262
  VolumeModel.deleted == False,
235
263
  )
236
264
  .options(selectinload(VolumeModel.user))
237
- .options(selectinload(VolumeModel.instances))
265
+ .options(selectinload(VolumeModel.attachments))
238
266
  .execution_options(populate_existing=True)
239
267
  .with_for_update()
240
268
  )
241
269
  volume_models = res.scalars().unique().all()
242
270
  for volume_model in volume_models:
243
- if len(volume_model.instances) > 0:
271
+ if len(volume_model.attachments) > 0:
244
272
  raise ServerClientError(
245
273
  f"Failed to delete volume {volume_model.name}. Volume is in use."
246
274
  )
@@ -270,6 +298,15 @@ def volume_model_to_volume(volume_model: VolumeModel) -> Volume:
270
298
  # Initially VolumeProvisionigData lacked backend
271
299
  if vpd is not None and vpd.backend is None:
272
300
  vpd.backend = configuration.backend
301
+ attachments = []
302
+ for volume_attachment_model in volume_model.attachments:
303
+ instance = volume_attachment_model.instance
304
+ attachments.append(
305
+ VolumeAttachment(
306
+ instance=instance_model_to_volume_instance(instance),
307
+ attachment_data=get_attachment_data(volume_attachment_model),
308
+ )
309
+ )
273
310
  return Volume(
274
311
  name=volume_model.name,
275
312
  project_name=volume_model.project.name,
@@ -282,6 +319,7 @@ def volume_model_to_volume(volume_model: VolumeModel) -> Volume:
282
319
  deleted=volume_model.deleted,
283
320
  volume_id=vpd.volume_id if vpd is not None else None,
284
321
  provisioning_data=vpd,
322
+ attachments=attachments,
285
323
  attachment_data=vad,
286
324
  id=volume_model.id,
287
325
  )
@@ -303,6 +341,27 @@ def get_volume_attachment_data(volume_model: VolumeModel) -> Optional[VolumeAtta
303
341
  return VolumeAttachmentData.__response__.parse_raw(volume_model.volume_attachment_data)
304
342
 
305
343
 
344
+ def get_attachment_data(
345
+ volume_attachment_model: VolumeAttachmentModel,
346
+ ) -> Optional[VolumeAttachmentData]:
347
+ if volume_attachment_model.attachment_data is None:
348
+ return None
349
+ return VolumeAttachmentData.__response__.parse_raw(volume_attachment_model.attachment_data)
350
+
351
+
352
+ def instance_model_to_volume_instance(instance_model: InstanceModel) -> VolumeInstance:
353
+ instance_id = None
354
+ jpd = get_instance_provisioning_data(instance_model)
355
+ if jpd is not None:
356
+ instance_id = jpd.instance_id
357
+ return VolumeInstance(
358
+ name=instance_model.name,
359
+ fleet_name=instance_model.fleet.name if instance_model.fleet else None,
360
+ instance_num=instance_model.instance_num,
361
+ instance_id=instance_id,
362
+ )
363
+
364
+
306
365
  async def generate_volume_name(session: AsyncSession, project: ProjectModel) -> str:
307
366
  volume_models = await list_project_volume_models(session=session, project=project)
308
367
  names = {v.name for v in volume_models}
@@ -34,12 +34,15 @@ DB_MAX_OVERFLOW = int(os.getenv("DSTACK_DB_MAX_OVERFLOW", 10))
34
34
  SERVER_CONFIG_DISABLED = os.getenv("DSTACK_SERVER_CONFIG_DISABLED") is not None
35
35
  SERVER_CONFIG_ENABLED = not SERVER_CONFIG_DISABLED
36
36
 
37
+ # TODO: add s3/aws prefix
37
38
  SERVER_BUCKET = os.getenv("DSTACK_SERVER_BUCKET")
38
39
  SERVER_BUCKET_REGION = os.getenv("DSTACK_SERVER_BUCKET_REGION")
39
40
 
40
41
  SERVER_CLOUDWATCH_LOG_GROUP = os.getenv("DSTACK_SERVER_CLOUDWATCH_LOG_GROUP")
41
42
  SERVER_CLOUDWATCH_LOG_REGION = os.getenv("DSTACK_SERVER_CLOUDWATCH_LOG_REGION")
42
43
 
44
+ SERVER_GCP_LOGGING_PROJECT = os.getenv("DSTACK_SERVER_GCP_LOGGING_PROJECT")
45
+
43
46
  SERVER_METRICS_TTL_SECONDS = int(os.getenv("DSTACK_SERVER_METRICS_TTL_SECONDS", 3600))
44
47
 
45
48
  DEFAULT_PROJECT_NAME = "main"
@@ -1,3 +1,3 @@
1
1
  <!doctype html><html lang="en"><head><meta charset="utf-8"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><title>dstack</title><meta name="description" content="Get GPUs at the best prices and availability from a wide range of providers. No cloud account of your own is required.
2
2
  "/><link rel="preconnect" href="https://fonts.googleapis.com"><link rel="preconnect" href="https://fonts.gstatic.com" crossorigin><link href="https://fonts.googleapis.com/css2?family=Roboto:ital,wght@0,100;0,300;0,400;0,500;0,700;0,900;1,100;1,300;1,400;1,500;1,700;1,900&display=swap" rel="stylesheet"><meta name="og:title" content="dstack"><meta name="og:type" content="article"><meta name="og:image" content="/splash_thumbnail.png"><meta name="og:description" content="Get GPUs at the best prices and availability from a wide range of providers. No cloud account of your own is required.
3
- "><link rel="icon" type="image/x-icon" href="/assets/favicon.ico"><link rel="icon" type="image/png" sizes="16x16" href="/assets/favicon-16x16.png"><link rel="icon" type="image/png" sizes="32x32" href="/assets/favicon-32x32.png"><link rel="icon" type="image/png" sizes="48x48" href="/assets/favicon-48x48.png"><link rel="manifest" href="/assets/manifest.webmanifest"><meta name="mobile-web-app-capable" content="yes"><meta name="theme-color" content="#fff"><meta name="application-name" content="dstackai"><link rel="apple-touch-icon" sizes="57x57" href="/assets/apple-touch-icon-57x57.png"><link rel="apple-touch-icon" sizes="60x60" href="/assets/apple-touch-icon-60x60.png"><link rel="apple-touch-icon" sizes="72x72" href="/assets/apple-touch-icon-72x72.png"><link rel="apple-touch-icon" sizes="76x76" href="/assets/apple-touch-icon-76x76.png"><link rel="apple-touch-icon" sizes="114x114" href="/assets/apple-touch-icon-114x114.png"><link rel="apple-touch-icon" sizes="120x120" href="/assets/apple-touch-icon-120x120.png"><link rel="apple-touch-icon" sizes="144x144" href="/assets/apple-touch-icon-144x144.png"><link rel="apple-touch-icon" sizes="152x152" href="/assets/apple-touch-icon-152x152.png"><link rel="apple-touch-icon" sizes="167x167" href="/assets/apple-touch-icon-167x167.png"><link rel="apple-touch-icon" sizes="180x180" href="/assets/apple-touch-icon-180x180.png"><link rel="apple-touch-icon" sizes="1024x1024" href="/assets/apple-touch-icon-1024x1024.png"><meta name="apple-mobile-web-app-capable" content="yes"><meta name="apple-mobile-web-app-status-bar-style" content="black-translucent"><meta name="apple-mobile-web-app-title" content="dstackai"><link rel="apple-touch-startup-image" media="(device-width: 320px) and (device-height: 568px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-640x1136.png"><link rel="apple-touch-startup-image" media="(device-width: 320px) and (device-height: 568px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1136x640.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 667px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-750x1334.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 667px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1334x750.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 812px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1125x2436.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 812px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2436x1125.png"><link rel="apple-touch-startup-image" media="(device-width: 390px) and (device-height: 844px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1170x2532.png"><link rel="apple-touch-startup-image" media="(device-width: 390px) and (device-height: 844px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2532x1170.png"><link rel="apple-touch-startup-image" media="(device-width: 393px) and (device-height: 852px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1179x2556.png"><link rel="apple-touch-startup-image" media="(device-width: 393px) and (device-height: 852px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2556x1179.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-828x1792.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1792x828.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1242x2688.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2688x1242.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 736px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1242x2208.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 736px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2208x1242.png"><link rel="apple-touch-startup-image" media="(device-width: 428px) and (device-height: 926px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1284x2778.png"><link rel="apple-touch-startup-image" media="(device-width: 428px) and (device-height: 926px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2778x1284.png"><link rel="apple-touch-startup-image" media="(device-width: 430px) and (device-height: 932px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1290x2796.png"><link rel="apple-touch-startup-image" media="(device-width: 430px) and (device-height: 932px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2796x1290.png"><link rel="apple-touch-startup-image" media="(device-width: 744px) and (device-height: 1133px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1488x2266.png"><link rel="apple-touch-startup-image" media="(device-width: 744px) and (device-height: 1133px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2266x1488.png"><link rel="apple-touch-startup-image" media="(device-width: 768px) and (device-height: 1024px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1536x2048.png"><link rel="apple-touch-startup-image" media="(device-width: 768px) and (device-height: 1024px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2048x1536.png"><link rel="apple-touch-startup-image" media="(device-width: 810px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1620x2160.png"><link rel="apple-touch-startup-image" media="(device-width: 810px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2160x1620.png"><link rel="apple-touch-startup-image" media="(device-width: 820px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1640x2160.png"><link rel="apple-touch-startup-image" media="(device-width: 820px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2160x1640.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1194px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1668x2388.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1194px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2388x1668.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1112px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1668x2224.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1112px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2224x1668.png"><link rel="apple-touch-startup-image" media="(device-width: 1024px) and (device-height: 1366px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-2048x2732.png"><link rel="apple-touch-startup-image" media="(device-width: 1024px) and (device-height: 1366px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2732x2048.png"><meta name="msapplication-TileColor" content="#fff"><meta name="msapplication-TileImage" content="/assets/mstile-144x144.png"><meta name="msapplication-config" content="/assets/browserconfig.xml"><link rel="yandex-tableau-widget" href="/assets/yandex-browser-manifest.json"><script defer="defer" src="/main-2ac66bfcbd2e39830b88.js"></script><link href="/main-ad5150a441de98cd8987.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div class="b-page-header" id="header"></div><div id="root"></div></body></html>
3
+ "><link rel="icon" type="image/x-icon" href="/assets/favicon.ico"><link rel="icon" type="image/png" sizes="16x16" href="/assets/favicon-16x16.png"><link rel="icon" type="image/png" sizes="32x32" href="/assets/favicon-32x32.png"><link rel="icon" type="image/png" sizes="48x48" href="/assets/favicon-48x48.png"><link rel="manifest" href="/assets/manifest.webmanifest"><meta name="mobile-web-app-capable" content="yes"><meta name="theme-color" content="#fff"><meta name="application-name" content="dstackai"><link rel="apple-touch-icon" sizes="57x57" href="/assets/apple-touch-icon-57x57.png"><link rel="apple-touch-icon" sizes="60x60" href="/assets/apple-touch-icon-60x60.png"><link rel="apple-touch-icon" sizes="72x72" href="/assets/apple-touch-icon-72x72.png"><link rel="apple-touch-icon" sizes="76x76" href="/assets/apple-touch-icon-76x76.png"><link rel="apple-touch-icon" sizes="114x114" href="/assets/apple-touch-icon-114x114.png"><link rel="apple-touch-icon" sizes="120x120" href="/assets/apple-touch-icon-120x120.png"><link rel="apple-touch-icon" sizes="144x144" href="/assets/apple-touch-icon-144x144.png"><link rel="apple-touch-icon" sizes="152x152" href="/assets/apple-touch-icon-152x152.png"><link rel="apple-touch-icon" sizes="167x167" href="/assets/apple-touch-icon-167x167.png"><link rel="apple-touch-icon" sizes="180x180" href="/assets/apple-touch-icon-180x180.png"><link rel="apple-touch-icon" sizes="1024x1024" href="/assets/apple-touch-icon-1024x1024.png"><meta name="apple-mobile-web-app-capable" content="yes"><meta name="apple-mobile-web-app-status-bar-style" content="black-translucent"><meta name="apple-mobile-web-app-title" content="dstackai"><link rel="apple-touch-startup-image" media="(device-width: 320px) and (device-height: 568px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-640x1136.png"><link rel="apple-touch-startup-image" media="(device-width: 320px) and (device-height: 568px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1136x640.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 667px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-750x1334.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 667px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1334x750.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 812px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1125x2436.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 812px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2436x1125.png"><link rel="apple-touch-startup-image" media="(device-width: 390px) and (device-height: 844px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1170x2532.png"><link rel="apple-touch-startup-image" media="(device-width: 390px) and (device-height: 844px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2532x1170.png"><link rel="apple-touch-startup-image" media="(device-width: 393px) and (device-height: 852px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1179x2556.png"><link rel="apple-touch-startup-image" media="(device-width: 393px) and (device-height: 852px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2556x1179.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-828x1792.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1792x828.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1242x2688.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2688x1242.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 736px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1242x2208.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 736px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2208x1242.png"><link rel="apple-touch-startup-image" media="(device-width: 428px) and (device-height: 926px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1284x2778.png"><link rel="apple-touch-startup-image" media="(device-width: 428px) and (device-height: 926px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2778x1284.png"><link rel="apple-touch-startup-image" media="(device-width: 430px) and (device-height: 932px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1290x2796.png"><link rel="apple-touch-startup-image" media="(device-width: 430px) and (device-height: 932px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2796x1290.png"><link rel="apple-touch-startup-image" media="(device-width: 744px) and (device-height: 1133px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1488x2266.png"><link rel="apple-touch-startup-image" media="(device-width: 744px) and (device-height: 1133px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2266x1488.png"><link rel="apple-touch-startup-image" media="(device-width: 768px) and (device-height: 1024px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1536x2048.png"><link rel="apple-touch-startup-image" media="(device-width: 768px) and (device-height: 1024px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2048x1536.png"><link rel="apple-touch-startup-image" media="(device-width: 810px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1620x2160.png"><link rel="apple-touch-startup-image" media="(device-width: 810px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2160x1620.png"><link rel="apple-touch-startup-image" media="(device-width: 820px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1640x2160.png"><link rel="apple-touch-startup-image" media="(device-width: 820px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2160x1640.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1194px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1668x2388.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1194px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2388x1668.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1112px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1668x2224.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1112px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2224x1668.png"><link rel="apple-touch-startup-image" media="(device-width: 1024px) and (device-height: 1366px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-2048x2732.png"><link rel="apple-touch-startup-image" media="(device-width: 1024px) and (device-height: 1366px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2732x2048.png"><meta name="msapplication-TileColor" content="#fff"><meta name="msapplication-TileImage" content="/assets/mstile-144x144.png"><meta name="msapplication-config" content="/assets/browserconfig.xml"><link rel="yandex-tableau-widget" href="/assets/yandex-browser-manifest.json"><script defer="defer" src="/main-fe8fd9db55df8d10e648.js"></script><link href="/main-7510e71dfa9749a4e70e.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div class="b-page-header" id="header"></div><div id="root"></div></body></html>