dstack 0.19.19__py3-none-any.whl → 0.19.21__py3-none-any.whl

dstack/_internal/core/backends/__init__.py

@@ -1,65 +0,0 @@
-from dstack._internal.core.backends.base.compute import (
-    ComputeWithCreateInstanceSupport,
-    ComputeWithGatewaySupport,
-    ComputeWithMultinodeSupport,
-    ComputeWithPlacementGroupSupport,
-    ComputeWithPrivateGatewaySupport,
-    ComputeWithReservationSupport,
-    ComputeWithVolumeSupport,
-)
-from dstack._internal.core.backends.base.configurator import Configurator
-from dstack._internal.core.backends.configurators import list_available_configurator_classes
-from dstack._internal.core.backends.local.compute import LocalCompute
-from dstack._internal.core.models.backends.base import BackendType
-from dstack._internal.settings import LOCAL_BACKEND_ENABLED
-
-
-def _get_backends_with_compute_feature(
-    configurator_classes: list[type[Configurator]],
-    compute_feature_class: type,
-) -> list[BackendType]:
-    backend_types_and_computes = [
-        (configurator_class.TYPE, configurator_class.BACKEND_CLASS.COMPUTE_CLASS)
-        for configurator_class in configurator_classes
-    ]
-    if LOCAL_BACKEND_ENABLED:
-        backend_types_and_computes.append((BackendType.LOCAL, LocalCompute))
-    backend_types = []
-    for backend_type, compute_class in backend_types_and_computes:
-        if issubclass(compute_class, compute_feature_class):
-            backend_types.append(backend_type)
-    return backend_types
-
-
-_configurator_classes = list_available_configurator_classes()
-
-
-# The following backend lists do not include unavailable backends (i.e. backends missing deps).
-BACKENDS_WITH_CREATE_INSTANCE_SUPPORT = _get_backends_with_compute_feature(
-    configurator_classes=_configurator_classes,
-    compute_feature_class=ComputeWithCreateInstanceSupport,
-)
-BACKENDS_WITH_MULTINODE_SUPPORT = [BackendType.REMOTE] + _get_backends_with_compute_feature(
-    configurator_classes=_configurator_classes,
-    compute_feature_class=ComputeWithMultinodeSupport,
-)
-BACKENDS_WITH_PLACEMENT_GROUPS_SUPPORT = _get_backends_with_compute_feature(
-    configurator_classes=_configurator_classes,
-    compute_feature_class=ComputeWithPlacementGroupSupport,
-)
-BACKENDS_WITH_RESERVATION_SUPPORT = _get_backends_with_compute_feature(
-    configurator_classes=_configurator_classes,
-    compute_feature_class=ComputeWithReservationSupport,
-)
-BACKENDS_WITH_GATEWAY_SUPPORT = _get_backends_with_compute_feature(
-    configurator_classes=_configurator_classes,
-    compute_feature_class=ComputeWithGatewaySupport,
-)
-BACKENDS_WITH_PRIVATE_GATEWAY_SUPPORT = _get_backends_with_compute_feature(
-    configurator_classes=_configurator_classes,
-    compute_feature_class=ComputeWithPrivateGatewaySupport,
-)
-BACKENDS_WITH_VOLUMES_SUPPORT = _get_backends_with_compute_feature(
-    configurator_classes=_configurator_classes,
-    compute_feature_class=ComputeWithVolumeSupport,
-)

dstack/_internal/core/backends/cloudrift/api_client.py

@@ -155,8 +155,20 @@ class RiftClient:
         logger.debug("Terminating instance with request data: %s", request_data)
         response_data = self._make_request("instances/terminate", request_data)
         if isinstance(response_data, dict):
+            logger.debug("Terminating instance with response: %s", response_data)
             info = response_data.get("terminated", [])
-            return len(info) > 0
+            is_terminated = len(info) > 0
+            if not is_terminated:
+                # check if the instance is already terminated
+                instance_info = self.get_instance_by_id(instance_id)
+                is_terminated = instance_info is None or instance_info.get("status") == "Inactive"
+                logger.debug(
+                    "Instance %s is already terminated: %s response: %s",
+                    instance_id,
+                    is_terminated,
+                    instance_info,
+                )
+            return is_terminated
 
         return False
 

dstack/_internal/core/backends/features.py

@@ -0,0 +1,64 @@
+from dstack._internal.core.backends.base.compute import (
+    ComputeWithCreateInstanceSupport,
+    ComputeWithGatewaySupport,
+    ComputeWithMultinodeSupport,
+    ComputeWithPlacementGroupSupport,
+    ComputeWithPrivateGatewaySupport,
+    ComputeWithReservationSupport,
+    ComputeWithVolumeSupport,
+)
+from dstack._internal.core.backends.base.configurator import Configurator
+from dstack._internal.core.backends.configurators import list_available_configurator_classes
+from dstack._internal.core.backends.local.compute import LocalCompute
+from dstack._internal.core.models.backends.base import BackendType
+from dstack._internal.settings import LOCAL_BACKEND_ENABLED
+
+_configurator_classes = list_available_configurator_classes()
+
+
+def _get_backends_with_compute_feature(
+    configurator_classes: list[type[Configurator]],
+    compute_feature_class: type,
+) -> list[BackendType]:
+    backend_types_and_computes = [
+        (configurator_class.TYPE, configurator_class.BACKEND_CLASS.COMPUTE_CLASS)
+        for configurator_class in configurator_classes
+    ]
+    if LOCAL_BACKEND_ENABLED:
+        backend_types_and_computes.append((BackendType.LOCAL, LocalCompute))
+    backend_types = []
+    for backend_type, compute_class in backend_types_and_computes:
+        if issubclass(compute_class, compute_feature_class):
+            backend_types.append(backend_type)
+    return backend_types
+
+
+# The following backend lists do not include unavailable backends (i.e. backends missing deps).
+BACKENDS_WITH_CREATE_INSTANCE_SUPPORT = _get_backends_with_compute_feature(
+    configurator_classes=_configurator_classes,
+    compute_feature_class=ComputeWithCreateInstanceSupport,
+)
+BACKENDS_WITH_MULTINODE_SUPPORT = [BackendType.REMOTE] + _get_backends_with_compute_feature(
+    configurator_classes=_configurator_classes,
+    compute_feature_class=ComputeWithMultinodeSupport,
+)
+BACKENDS_WITH_PLACEMENT_GROUPS_SUPPORT = _get_backends_with_compute_feature(
+    configurator_classes=_configurator_classes,
+    compute_feature_class=ComputeWithPlacementGroupSupport,
+)
+BACKENDS_WITH_RESERVATION_SUPPORT = _get_backends_with_compute_feature(
+    configurator_classes=_configurator_classes,
+    compute_feature_class=ComputeWithReservationSupport,
+)
+BACKENDS_WITH_GATEWAY_SUPPORT = _get_backends_with_compute_feature(
+    configurator_classes=_configurator_classes,
+    compute_feature_class=ComputeWithGatewaySupport,
+)
+BACKENDS_WITH_PRIVATE_GATEWAY_SUPPORT = _get_backends_with_compute_feature(
+    configurator_classes=_configurator_classes,
+    compute_feature_class=ComputeWithPrivateGatewaySupport,
+)
+BACKENDS_WITH_VOLUMES_SUPPORT = _get_backends_with_compute_feature(
+    configurator_classes=_configurator_classes,
+    compute_feature_class=ComputeWithVolumeSupport,
+)

dstack/_internal/core/backends/oci/resources.py

@@ -26,7 +26,7 @@ from dstack import version
 from dstack._internal.core.backends.oci.region import OCIRegionClient
 from dstack._internal.core.errors import BackendError
 from dstack._internal.core.models.instances import InstanceOffer
-from dstack._internal.utils.common import split_chunks
+from dstack._internal.utils.common import batched
 from dstack._internal.utils.logging import get_logger
 
 logger = get_logger(__name__)
@@ -667,21 +667,21 @@ def add_security_group_rules(
     security_group_id: str, rules: Iterable[SecurityRule], client: oci.core.VirtualNetworkClient
 ) -> None:
     rules_details = map(SecurityRule.to_sdk_add_rule_details, rules)
-    for chunk in split_chunks(rules_details, ADD_SECURITY_RULES_MAX_CHUNK_SIZE):
+    for batch in batched(rules_details, ADD_SECURITY_RULES_MAX_CHUNK_SIZE):
         client.add_network_security_group_security_rules(
             security_group_id,
-            oci.core.models.AddNetworkSecurityGroupSecurityRulesDetails(security_rules=chunk),
+            oci.core.models.AddNetworkSecurityGroupSecurityRulesDetails(security_rules=batch),
         )
 
 
 def remove_security_group_rules(
     security_group_id: str, rule_ids: Iterable[str], client: oci.core.VirtualNetworkClient
 ) -> None:
-    for chunk in split_chunks(rule_ids, REMOVE_SECURITY_RULES_MAX_CHUNK_SIZE):
+    for batch in batched(rule_ids, REMOVE_SECURITY_RULES_MAX_CHUNK_SIZE):
         client.remove_network_security_group_security_rules(
             security_group_id,
             oci.core.models.RemoveNetworkSecurityGroupSecurityRulesDetails(
-                security_rule_ids=chunk
+                security_rule_ids=batch
             ),
         )
 

dstack/_internal/core/compatibility/fleets.py

@@ -57,6 +57,8 @@ def get_fleet_spec_excludes(fleet_spec: FleetSpec) -> Optional[IncludeExcludeDic
         profile_excludes.add("startup_order")
     if profile.stop_criteria is None:
         profile_excludes.add("stop_criteria")
+    if profile.schedule is None:
+        profile_excludes.add("schedule")
     if configuration_excludes:
         spec_excludes["configuration"] = configuration_excludes
     if profile_excludes:

dstack/_internal/core/compatibility/runs.py

@@ -126,6 +126,10 @@ def get_run_spec_excludes(run_spec: RunSpec) -> IncludeExcludeDictType:
         configuration_excludes["files"] = True
     if not run_spec.file_archives:
         spec_excludes["file_archives"] = True
+    if configuration.schedule is None:
+        configuration_excludes["schedule"] = True
+    if profile is not None and profile.schedule is None:
+        profile_excludes.add("schedule")
 
     if configuration_excludes:
         spec_excludes["configuration"] = configuration_excludes

dstack/_internal/core/models/profiles.py

@@ -8,6 +8,7 @@ from typing_extensions import Annotated, Literal
 from dstack._internal.core.models.backends.base import BackendType
 from dstack._internal.core.models.common import CoreModel, Duration
 from dstack._internal.utils.common import list_enum_values_for_annotation
+from dstack._internal.utils.cron import validate_cron
 from dstack._internal.utils.json_utils import pydantic_orjson_dumps_with_indent
 from dstack._internal.utils.tags import tags_validator
 
@@ -167,6 +168,38 @@ class UtilizationPolicy(CoreModel):
         return v
 
 
+class Schedule(CoreModel):
+    cron: Annotated[
+        Union[List[str], str],
+        Field(
+            description=(
+                "A cron expression or a list of cron expressions specifying the UTC time when the run needs to be started"
+            )
+        ),
+    ]
+
+    @validator("cron")
+    def _validate_cron(cls, v: Union[List[str], str]) -> List[str]:
+        if isinstance(v, str):
+            values = [v]
+        else:
+            values = v
+        if len(values) == 0:
+            raise ValueError("At least one cron expression must be specified")
+        for value in values:
+            validate_cron(value)
+        return values
+
+    @property
+    def crons(self) -> List[str]:
+        """
+        Access `cron` attribute as a list.
+        """
+        if isinstance(self.cron, str):
+            return [self.cron]
+        return self.cron
+
+
 class ProfileParams(CoreModel):
     backends: Annotated[
         Optional[List[BackendType]],
@@ -281,6 +314,10 @@ class ProfileParams(CoreModel):
             )
         ),
     ] = None
+    schedule: Annotated[
+        Optional[Schedule],
+        Field(description=("The schedule for starting the run at specified time")),
+    ] = None
     fleets: Annotated[
         Optional[list[str]], Field(description="The fleets considered for reuse")
     ] = None

dstack/_internal/server/app.py

@@ -13,6 +13,7 @@ from fastapi.datastructures import URL
 from fastapi.responses import HTMLResponse, RedirectResponse
 from fastapi.staticfiles import StaticFiles
 from prometheus_client import Counter, Histogram
+from sentry_sdk.types import SamplingContext
 
 from dstack._internal.cli.utils.common import console
 from dstack._internal.core.errors import ForbiddenError, ServerClientError
@@ -81,16 +82,6 @@ REQUEST_DURATION = Histogram(
 
 
 def create_app() -> FastAPI:
-    if settings.SENTRY_DSN is not None:
-        sentry_sdk.init(
-            dsn=settings.SENTRY_DSN,
-            release=DSTACK_VERSION,
-            environment=settings.SERVER_ENVIRONMENT,
-            enable_tracing=True,
-            traces_sample_rate=settings.SENTRY_TRACES_SAMPLE_RATE,
-            profiles_sample_rate=settings.SENTRY_PROFILES_SAMPLE_RATE,
-        )
-
     app = FastAPI(
         docs_url="/api/docs",
         lifespan=lifespan,
@@ -102,6 +93,15 @@ def create_app() -> FastAPI:
 @asynccontextmanager
 async def lifespan(app: FastAPI):
     configure_logging()
+    if settings.SENTRY_DSN is not None:
+        sentry_sdk.init(
+            dsn=settings.SENTRY_DSN,
+            release=DSTACK_VERSION,
+            environment=settings.SERVER_ENVIRONMENT,
+            enable_tracing=True,
+            traces_sampler=_sentry_traces_sampler,
+            profiles_sample_rate=settings.SENTRY_PROFILES_SAMPLE_RATE,
+        )
     server_executor = ThreadPoolExecutor(max_workers=settings.SERVER_EXECUTOR_MAX_WORKERS)
     asyncio.get_running_loop().set_default_executor(server_executor)
     await migrate()
@@ -379,3 +379,15 @@ def _print_dstack_logo():
 ╰━━┻━━┻╯╱╰╯╰━━┻╯
 [/]"""
     )
+
+
+def _sentry_traces_sampler(sampling_context: SamplingContext) -> float:
+    parent_sampling_decision = sampling_context["parent_sampled"]
+    if parent_sampling_decision is not None:
+        return float(parent_sampling_decision)
+    transaction_context = sampling_context["transaction_context"]
+    name = transaction_context.get("name")
+    if name is not None:
+        if name.startswith("background."):
+            return settings.SENTRY_TRACES_BACKGROUND_SAMPLE_RATE
+    return settings.SENTRY_TRACES_SAMPLE_RATE

dstack/_internal/server/background/__init__.py

@@ -79,6 +79,11 @@ def start_background_tasks() -> AsyncIOScheduler:
         process_idle_volumes, IntervalTrigger(seconds=60, jitter=10), max_instances=1
     )
     _scheduler.add_job(process_placement_groups, IntervalTrigger(seconds=30, jitter=5))
+    _scheduler.add_job(
+        process_fleets,
+        IntervalTrigger(seconds=10, jitter=2),
+        max_instances=1,
+    )
     for replica in range(settings.SERVER_BACKGROUND_PROCESSING_FACTOR):
         # Add multiple copies of tasks if requested.
         # max_instances=1 for additional copies to avoid running too many tasks.
@@ -113,11 +118,5 @@ def start_background_tasks() -> AsyncIOScheduler:
             kwargs={"batch_size": 5},
             max_instances=2 if replica == 0 else 1,
         )
-        _scheduler.add_job(
-            process_fleets,
-            IntervalTrigger(seconds=10, jitter=2),
-            kwargs={"batch_size": 5},
-            max_instances=2 if replica == 0 else 1,
-        )
     _scheduler.start()
     return _scheduler

dstack/_internal/server/background/tasks/process_fleets.py

@@ -1,36 +1,37 @@
-import asyncio
 from datetime import timedelta
+from typing import List
 
-from sqlalchemy import select
+from sqlalchemy import select, update
 from sqlalchemy.ext.asyncio import AsyncSession
-from sqlalchemy.orm import joinedload
+from sqlalchemy.orm import joinedload, load_only
 
 from dstack._internal.core.models.fleets import FleetStatus
 from dstack._internal.server.db import get_db, get_session_ctx
-from dstack._internal.server.models import FleetModel
+from dstack._internal.server.models import (
+    FleetModel,
+    InstanceModel,
+    JobModel,
+    PlacementGroupModel,
+    RunModel,
+)
 from dstack._internal.server.services.fleets import (
     is_fleet_empty,
     is_fleet_in_use,
 )
 from dstack._internal.server.services.locking import get_locker
-from dstack._internal.server.services.placement import schedule_fleet_placement_groups_deletion
+from dstack._internal.server.utils import sentry_utils
 from dstack._internal.utils.common import get_current_datetime
 from dstack._internal.utils.logging import get_logger
 
 logger = get_logger(__name__)
 
 
+BATCH_SIZE = 10
 MIN_PROCESSING_INTERVAL = timedelta(seconds=30)
 
 
-async def process_fleets(batch_size: int = 1):
-    tasks = []
-    for _ in range(batch_size):
-        tasks.append(_process_next_fleet())
-    await asyncio.gather(*tasks)
-
-
-async def _process_next_fleet():
+@sentry_utils.instrument_background_task
+async def process_fleets():
     lock, lockset = get_locker(get_db().dialect_name).get_lockset(FleetModel.__tablename__)
     async with get_session_ctx() as session:
         async with lock:
@@ -40,51 +41,64 @@ async def _process_next_fleet():
                     FleetModel.deleted == False,
                     FleetModel.id.not_in(lockset),
                     FleetModel.last_processed_at
-                    < get_current_datetime().replace(tzinfo=None) - MIN_PROCESSING_INTERVAL,
+                    < get_current_datetime() - MIN_PROCESSING_INTERVAL,
                 )
+                .options(load_only(FleetModel.id))
                 .order_by(FleetModel.last_processed_at.asc())
-                .limit(1)
+                .limit(BATCH_SIZE)
                 .with_for_update(skip_locked=True, key_share=True)
             )
-            fleet_model = res.scalar()
-            if fleet_model is None:
-                return
-            lockset.add(fleet_model.id)
+            fleet_models = list(res.scalars().all())
+            fleet_ids = [fm.id for fm in fleet_models]
+            for fleet_id in fleet_ids:
+                lockset.add(fleet_id)
         try:
-            fleet_model_id = fleet_model.id
-            await _process_fleet(session=session, fleet_model=fleet_model)
+            await _process_fleets(session=session, fleet_models=fleet_models)
         finally:
-            lockset.difference_update([fleet_model_id])
+            lockset.difference_update(fleet_ids)
 
 
-async def _process_fleet(session: AsyncSession, fleet_model: FleetModel):
-    logger.debug("Processing fleet %s", fleet_model.name)
+async def _process_fleets(session: AsyncSession, fleet_models: List[FleetModel]):
+    fleet_ids = [fm.id for fm in fleet_models]
     # Refetch to load related attributes.
-    # joinedload produces LEFT OUTER JOIN that can't be used with FOR UPDATE.
     res = await session.execute(
         select(FleetModel)
-        .where(FleetModel.id == fleet_model.id)
-        .options(joinedload(FleetModel.project))
-        .options(joinedload(FleetModel.instances))
-        .options(joinedload(FleetModel.runs))
+        .where(FleetModel.id.in_(fleet_ids))
+        .options(joinedload(FleetModel.instances).load_only(InstanceModel.deleted))
+        .options(
+            joinedload(FleetModel.instances).joinedload(InstanceModel.jobs).load_only(JobModel.id)
+        )
+        .options(joinedload(FleetModel.runs).load_only(RunModel.status))
         .execution_options(populate_existing=True)
     )
-    fleet_model = res.unique().scalar_one()
-    await _autodelete_fleet(session=session, fleet_model=fleet_model)
+    fleet_models = list(res.unique().scalars().all())
+
+    deleted_fleets_ids = []
+    now = get_current_datetime()
+    for fleet_model in fleet_models:
+        deleted = _autodelete_fleet(fleet_model)
+        if deleted:
+            deleted_fleets_ids.append(fleet_model.id)
+        fleet_model.last_processed_at = now
+
+    await session.execute(
+        update(PlacementGroupModel)
+        .where(
+            PlacementGroupModel.fleet_id.in_(deleted_fleets_ids),
+        )
+        .values(fleet_deleted=True)
+    )
+    await session.commit()
 
 
-async def _autodelete_fleet(session: AsyncSession, fleet_model: FleetModel):
+def _autodelete_fleet(fleet_model: FleetModel) -> bool:
     # Currently all empty fleets are autodeleted.
     # TODO: If fleets with `nodes: 0..` are supported, their deletion should be skipped.
     if is_fleet_in_use(fleet_model) or not is_fleet_empty(fleet_model):
-        fleet_model.last_processed_at = get_current_datetime()
-        await session.commit()
-        return
+        return False
 
     logger.info("Automatic cleanup of an empty fleet %s", fleet_model.name)
     fleet_model.status = FleetStatus.TERMINATED
     fleet_model.deleted = True
-    fleet_model.last_processed_at = get_current_datetime()
-    await schedule_fleet_placement_groups_deletion(session=session, fleet_id=fleet_model.id)
-    await session.commit()
     logger.info("Fleet %s deleted", fleet_model.name)
+    return True

dstack/_internal/server/background/tasks/process_gateways.py

@@ -17,6 +17,7 @@ from dstack._internal.server.services.gateways import (
 )
 from dstack._internal.server.services.locking import advisory_lock_ctx, get_locker
 from dstack._internal.server.services.logging import fmt
+from dstack._internal.server.utils import sentry_utils
 from dstack._internal.utils.common import get_current_datetime
 from dstack._internal.utils.logging import get_logger
 
@@ -28,6 +29,7 @@ async def process_gateways_connections():
     await _process_active_connections()
 
 
+@sentry_utils.instrument_background_task
 async def process_gateways():
     lock, lockset = get_locker(get_db().dialect_name).get_lockset(GatewayModel.__tablename__)
     async with get_session_ctx() as session:
@@ -110,7 +112,6 @@ async def _process_connection(conn: GatewayConnection):
 async def _process_submitted_gateway(session: AsyncSession, gateway_model: GatewayModel):
     logger.info("%s: started gateway provisioning", fmt(gateway_model))
     # Refetch to load related attributes.
-    # joinedload produces LEFT OUTER JOIN that can't be used with FOR UPDATE.
     res = await session.execute(
         select(GatewayModel)
         .where(GatewayModel.id == gateway_model.id)
@@ -157,7 +158,6 @@ async def _process_provisioning_gateway(
     session: AsyncSession, gateway_model: GatewayModel
 ) -> None:
     # Refetch to load related attributes.
-    # joinedload produces LEFT OUTER JOIN that can't be used with FOR UPDATE.
     res = await session.execute(
         select(GatewayModel)
         .where(GatewayModel.id == gateway_model.id)

dstack/_internal/server/background/tasks/process_idle_volumes.py

@@ -10,13 +10,14 @@ from dstack._internal.core.errors import BackendNotAvailable
 from dstack._internal.core.models.profiles import parse_duration
 from dstack._internal.core.models.volumes import VolumeStatus
 from dstack._internal.server.db import get_db, get_session_ctx
-from dstack._internal.server.models import ProjectModel, VolumeModel
+from dstack._internal.server.models import ProjectModel, UserModel, VolumeModel
 from dstack._internal.server.services import backends as backends_services
 from dstack._internal.server.services.locking import get_locker
 from dstack._internal.server.services.volumes import (
     get_volume_configuration,
     volume_model_to_volume,
 )
+from dstack._internal.server.utils import sentry_utils
 from dstack._internal.utils import common
 from dstack._internal.utils.common import get_current_datetime
 from dstack._internal.utils.logging import get_logger
@@ -24,6 +25,7 @@ from dstack._internal.utils.logging import get_logger
 logger = get_logger(__name__)
 
 
+@sentry_utils.instrument_background_task
 async def process_idle_volumes():
     lock, lockset = get_locker(get_db().dialect_name).get_lockset(VolumeModel.__tablename__)
     async with get_session_ctx() as session:
@@ -49,7 +51,7 @@ async def process_idle_volumes():
             select(VolumeModel)
             .where(VolumeModel.id.in_(volume_ids))
             .options(joinedload(VolumeModel.project).joinedload(ProjectModel.backends))
-            .options(joinedload(VolumeModel.user))
+            .options(joinedload(VolumeModel.user).load_only(UserModel.name))
             .options(joinedload(VolumeModel.attachments))
             .execution_options(populate_existing=True)
         )
@@ -82,8 +84,7 @@ def _should_delete_volume(volume: VolumeModel) -> bool:
 
 def _get_idle_time(volume: VolumeModel) -> datetime.timedelta:
     last_used = volume.last_job_processed_at or volume.created_at
-    last_used_utc = last_used.replace(tzinfo=datetime.timezone.utc)
-    idle_time = get_current_datetime() - last_used_utc
+    idle_time = get_current_datetime() - last_used
     return max(idle_time, datetime.timedelta(0))