snowflake-ml-python 1.8.5__py3-none-any.whl → 1.8.6__py3-none-any.whl

snowflake/ml/jobs/_utils/stage_utils.py

@@ -0,0 +1,119 @@
+import os
+import re
+from os import PathLike
+from pathlib import Path, PurePath
+from typing import Union
+
+from snowflake.ml._internal.utils import identifier
+
+PROTOCOL_NAME = "snow"
+_SNOWURL_PATH_RE = re.compile(
+    rf"^(?:(?:{PROTOCOL_NAME}://)?"
+    r"(?<!@)(?P<domain>\w+)/"
+    rf"(?P<name>(?:{identifier._SF_IDENTIFIER}\.){{,2}}{identifier._SF_IDENTIFIER})/)?"
+    r"(?P<path>versions(?:/(?P<version>[^/]+)(?:/(?P<relpath>.*))?)?)$"
+)
+
+_STAGEF_PATH_RE = re.compile(r"^@(?P<stage>~|%?\w+)(?:/(?P<relpath>[\w\-./]*))?$")
+
+
+class StagePath:
+    def __init__(self, path: str) -> None:
+        stage_match = _SNOWURL_PATH_RE.fullmatch(path) or _STAGEF_PATH_RE.fullmatch(path)
+        if not stage_match:
+            raise ValueError(f"{path} is not a valid stage path")
+        path = path.strip()
+        self._raw_path = path
+        relpath = stage_match.group("relpath")
+        start, _ = stage_match.span("relpath")
+        self._root = self._raw_path[0:start].rstrip("/") if relpath else self._raw_path.rstrip("/")
+        self._path = Path(relpath or "")
+
+    @property
+    def parent(self) -> "StagePath":
+        if self._path.parent == Path(""):
+            return StagePath(self._root)
+        else:
+            return StagePath(f"{self._root}/{self._path.parent}")
+
+    @property
+    def root(self) -> str:
+        return self._root
+
+    @property
+    def suffix(self) -> str:
+        return self._path.suffix
+
+    def _compose_path(self, path: Path) -> str:
+        # in pathlib, Path("") = "."
+        if path == Path(""):
+            return self.root
+        else:
+            return f"{self.root}/{path}"
+
+    def is_relative_to(self, path: Union[str, PathLike[str], "StagePath"]) -> bool:
+        stage_path = path if isinstance(path, StagePath) else StagePath(os.fspath(path))
+        if stage_path.root == self.root:
+            return self._path.is_relative_to(stage_path._path)
+        else:
+            return False
+
+    def relative_to(self, path: Union[str, PathLike[str], "StagePath"]) -> PurePath:
+        stage_path = path if isinstance(path, StagePath) else StagePath(os.fspath(path))
+        if self.root == stage_path.root:
+            return self._path.relative_to(stage_path._path)
+        raise ValueError(f"{self._raw_path} does not start with {stage_path._raw_path}")
+
+    def absolute(self) -> "StagePath":
+        return self
+
+    def as_posix(self) -> str:
+        return self._compose_path(self._path)
+
+    # TODO Add actual implementation https://snowflakecomputing.atlassian.net/browse/SNOW-2112795
+    def exists(self) -> bool:
+        return True
+
+    # TODO Add actual implementation https://snowflakecomputing.atlassian.net/browse/SNOW-2112795
+    def is_file(self) -> bool:
+        return True
+
+    # TODO Add actual implementation https://snowflakecomputing.atlassian.net/browse/SNOW-2112795
+    def is_dir(self) -> bool:
+        return True
+
+    def is_absolute(self) -> bool:
+        return True
+
+    def __str__(self) -> str:
+        return self.as_posix()
+
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, StagePath):
+            raise NotImplementedError
+        return bool(self.root == other.root and self._path == other._path)
+
+    def __fspath__(self) -> str:
+        return self._compose_path(self._path)
+
+    def joinpath(self, *args: Union[str, PathLike[str], "StagePath"]) -> "StagePath":
+        path = self
+        for arg in args:
+            path = path._make_child(arg)
+        return path
+
+    def _make_child(self, path: Union[str, PathLike[str], "StagePath"]) -> "StagePath":
+        stage_path = path if isinstance(path, StagePath) else StagePath(os.fspath(path))
+        if self.root == stage_path.root:
+            child_path = self._path.joinpath(stage_path._path)
+            return StagePath(self._compose_path(child_path))
+        else:
+            return stage_path
+
+
+def identify_stage_path(path: str) -> Union[StagePath, Path]:
+    try:
+        stage_path = StagePath(path)
+    except ValueError:
+        return Path(path)
+    return stage_path

snowflake/ml/jobs/_utils/types.py

@@ -2,18 +2,22 @@ from dataclasses import dataclass
 from pathlib import PurePath
 from typing import Literal, Optional, Union
 
+from snowflake.ml.jobs._utils import stage_utils
+
 JOB_STATUS = Literal[
     "PENDING",
     "RUNNING",
     "FAILED",
     "DONE",
+    "CANCELLING",
+    "CANCELLED",
     "INTERNAL_ERROR",
 ]
 
 
 @dataclass(frozen=True)
 class PayloadEntrypoint:
-    file_path: PurePath
+    file_path: Union[PurePath, stage_utils.StagePath]
     main_func: Optional[str]
 
 

snowflake/ml/jobs/decorators.py

@@ -7,7 +7,7 @@ from typing_extensions import ParamSpec
 from snowflake import snowpark
 from snowflake.ml._internal import telemetry
 from snowflake.ml.jobs import job as jb, manager as jm
-from snowflake.ml.jobs._utils import constants
+from snowflake.ml.jobs._utils import payload_utils
 
 _PROJECT = "MLJob"
 
@@ -25,7 +25,7 @@ def remote(
     query_warehouse: Optional[str] = None,
     env_vars: Optional[dict[str, str]] = None,
     target_instances: int = 1,
-    min_instances: int = 1,
+    min_instances: Optional[int] = None,
     enable_metrics: bool = False,
     database: Optional[str] = None,
     schema: Optional[str] = None,
@@ -42,8 +42,8 @@ def remote(
         query_warehouse: The query warehouse to use. Defaults to session warehouse.
         env_vars: Environment variables to set in container
         target_instances: The number of nodes in the job. If none specified, create a single node job.
-        min_instances: The minimum number of nodes required to start the job. If none specified, defaults to 1.
-            If set, the job will not start until the minimum number of nodes is available.
+        min_instances: The minimum number of nodes required to start the job. If none specified,
+            defaults to target_instances. If set, the job will not start until the minimum number of nodes is available.
         enable_metrics: Whether to enable metrics publishing for the job.
         database: The database to use for the job.
         schema: The schema to use for the job.
@@ -62,8 +62,7 @@ def remote(
 
         @functools.wraps(func)
         def wrapper(*args: _Args.args, **kwargs: _Args.kwargs) -> jb.MLJob[_ReturnValue]:
-            payload = functools.partial(func, *args, **kwargs)
-            setattr(payload, constants.IS_MLJOB_REMOTE_ATTR, True)
+            payload = payload_utils.create_function_payload(func, *args, **kwargs)
             job = jm._submit_job(
                 source=payload,
                 stage_name=stage_name,
@@ -77,7 +76,7 @@ def remote(
                 enable_metrics=enable_metrics,
                 database=database,
                 schema=schema,
-                session=session,
+                session=payload.session or session,
             )
             assert isinstance(job, jb.MLJob), f"Unexpected job type: {type(job)}"
             return job

snowflake/ml/jobs/job.py

@@ -14,7 +14,7 @@ from snowflake.snowpark import Row, context as sp_context
 from snowflake.snowpark.exceptions import SnowparkSQLException
 
 _PROJECT = "MLJob"
-TERMINAL_JOB_STATUSES = {"FAILED", "DONE", "INTERNAL_ERROR"}
+TERMINAL_JOB_STATUSES = {"FAILED", "DONE", "CANCELLED", "INTERNAL_ERROR"}
 
 T = TypeVar("T")
 
@@ -183,14 +183,14 @@ class MLJob(Generic[T]):
                 pool_info = _get_compute_pool_info(self._session, self._compute_pool)
                 if (pool_info.max_nodes - pool_info.active_nodes) < self.min_instances:
                     logger.warning(
-                        f"Compute pool busy ({pool_info.active_nodes}/{pool_info.max_nodes} nodes in use)."
-                        " Job execution may be delayed."
+                        f"Compute pool busy ({pool_info.active_nodes}/{pool_info.max_nodes} nodes in use, "
+                        f"{self.min_instances} nodes required). Job execution may be delayed."
                     )
                     warning_shown = True
             if timeout >= 0 and (elapsed := time.monotonic() - start_time) >= timeout:
                 raise TimeoutError(f"Job {self.name} did not complete within {elapsed} seconds")
             time.sleep(delay)
-            delay = min(delay * 2, constants.JOB_POLL_MAX_DELAY_SECONDS)  # Exponential backoff
+            delay = min(delay * 1.2, constants.JOB_POLL_MAX_DELAY_SECONDS)  # Exponential backoff
         return self.status
 
     @snowpark._internal.utils.private_preview(version="1.8.2")
@@ -338,16 +338,23 @@ def _get_head_instance_id(session: snowpark.Session, job_id: str) -> Optional[in
 
      Raises:
         RuntimeError: If the instances died or if some instances disappeared.
-
     """
+
+    target_instances = _get_target_instances(session, job_id)
+
+    if target_instances == 1:
+        return 0
+
     try:
         rows = session.sql("SHOW SERVICE INSTANCES IN SERVICE IDENTIFIER(?)", params=(job_id,)).collect()
     except SnowparkSQLException:
         # service may be deleted
         raise RuntimeError("Couldn’t retrieve instances")
+
     if not rows:
         return None
-    if _get_target_instances(session, job_id) > len(rows):
+
+    if target_instances > len(rows):
         raise RuntimeError("Couldn’t retrieve head instance due to missing instances.")
 
     # Sort by start_time first, then by instance_id
@@ -414,12 +421,20 @@ def _get_compute_pool_info(session: snowpark.Session, compute_pool: str) -> Row:
 
     Returns:
         Row: The compute pool information.
+
+    Raises:
+        ValueError: If the compute pool is not found.
     """
-    (pool_info,) = session.sql("SHOW COMPUTE POOLS LIKE ?", params=(compute_pool,)).collect()
-    return pool_info
+    try:
+        (pool_info,) = session.sql("SHOW COMPUTE POOLS LIKE ?", params=(compute_pool,)).collect()
+        return pool_info
+    except ValueError as e:
+        if "not enough values to unpack" in str(e):
+            raise ValueError(f"Compute pool '{compute_pool}' not found")
+        raise
 
 
 @telemetry.send_api_usage_telemetry(project=_PROJECT, func_params_to_log=["job_id"])
 def _get_target_instances(session: snowpark.Session, job_id: str) -> int:
     row = _get_service_info(session, job_id)
-    return int(row["target_instances"]) if row["target_instances"] else 0
+    return int(row["target_instances"])

snowflake/ml/jobs/manager.py

@@ -87,13 +87,15 @@ def get_job(job_id: str, session: Optional[snowpark.Session] = None) -> jb.MLJob
 @telemetry.send_api_usage_telemetry(project=_PROJECT)
 def delete_job(job: Union[str, jb.MLJob[Any]], session: Optional[snowpark.Session] = None) -> None:
     """Delete a job service from the backend. Status and logs will be lost."""
-    if isinstance(job, jb.MLJob):
-        job_id = job.id
-        session = job._session or session
-    else:
-        job_id = job
-    session = session or get_active_session()
-    session.sql("DROP SERVICE IDENTIFIER(?)", params=(job_id,)).collect()
+    job = job if isinstance(job, jb.MLJob) else get_job(job, session=session)
+    session = job._session
+    try:
+        stage_path = job._stage_path
+        session.sql(f"REMOVE {stage_path}/").collect()
+        logger.info(f"Successfully cleaned up stage files for job {job.id} at {stage_path}")
+    except Exception as e:
+        logger.warning(f"Failed to clean up stage files for job {job.id}: {e}")
+    session.sql("DROP SERVICE IDENTIFIER(?)", params=(job.id,)).collect()
 
 
 @telemetry.send_api_usage_telemetry(project=_PROJECT)
@@ -109,7 +111,7 @@ def submit_file(
     query_warehouse: Optional[str] = None,
     spec_overrides: Optional[dict[str, Any]] = None,
     target_instances: int = 1,
-    min_instances: int = 1,
+    min_instances: Optional[int] = None,
     enable_metrics: bool = False,
     database: Optional[str] = None,
     schema: Optional[str] = None,
@@ -129,7 +131,8 @@ def submit_file(
         query_warehouse: The query warehouse to use. Defaults to session warehouse.
         spec_overrides: Custom service specification overrides to apply.
         target_instances: The number of instances to use for the job. If none specified, single node job is created.
-        min_instances: The minimum number of nodes required to start the job. If none specified, defaults to 1.
+        min_instances: The minimum number of nodes required to start the job. If none specified,
+            defaults to target_instances. If set, the job will not start until the minimum number of nodes is available.
         enable_metrics: Whether to enable metrics publishing for the job.
         database: The database to use.
         schema: The schema to use.
@@ -171,7 +174,7 @@ def submit_directory(
     query_warehouse: Optional[str] = None,
     spec_overrides: Optional[dict[str, Any]] = None,
     target_instances: int = 1,
-    min_instances: int = 1,
+    min_instances: Optional[int] = None,
     enable_metrics: bool = False,
     database: Optional[str] = None,
     schema: Optional[str] = None,
@@ -192,7 +195,8 @@ def submit_directory(
         query_warehouse: The query warehouse to use. Defaults to session warehouse.
         spec_overrides: Custom service specification overrides to apply.
         target_instances: The number of instances to use for the job. If none specified, single node job is created.
-        min_instances: The minimum number of nodes required to start the job. If none specified, defaults to 1.
+        min_instances: The minimum number of nodes required to start the job. If none specified,
+            defaults to target_instances. If set, the job will not start until the minimum number of nodes is available.
         enable_metrics: Whether to enable metrics publishing for the job.
         database: The database to use.
         schema: The schema to use.
@@ -221,6 +225,72 @@ def submit_directory(
     )
 
 
+@telemetry.send_api_usage_telemetry(project=_PROJECT)
+def submit_from_stage(
+    source: str,
+    compute_pool: str,
+    *,
+    entrypoint: str,
+    stage_name: str,
+    args: Optional[list[str]] = None,
+    env_vars: Optional[dict[str, str]] = None,
+    pip_requirements: Optional[list[str]] = None,
+    external_access_integrations: Optional[list[str]] = None,
+    query_warehouse: Optional[str] = None,
+    spec_overrides: Optional[dict[str, Any]] = None,
+    target_instances: int = 1,
+    min_instances: Optional[int] = None,
+    enable_metrics: bool = False,
+    database: Optional[str] = None,
+    schema: Optional[str] = None,
+    session: Optional[snowpark.Session] = None,
+) -> jb.MLJob[None]:
+    """
+    Submit a directory containing Python script(s) as a job to the compute pool.
+
+    Args:
+        source: a stage path or a stage containing the job payload.
+        compute_pool: The compute pool to use for the job.
+        entrypoint: a stage path containing the entry point script inside the source directory.
+        stage_name: The name of the stage where the job payload will be uploaded.
+        args: A list of arguments to pass to the job.
+        env_vars: Environment variables to set in container
+        pip_requirements: A list of pip requirements for the job.
+        external_access_integrations: A list of external access integrations.
+        query_warehouse: The query warehouse to use. Defaults to session warehouse.
+        spec_overrides: Custom service specification overrides to apply.
+        target_instances: The number of instances to use for the job. If none specified, single node job is created.
+        min_instances: The minimum number of nodes required to start the job. If none specified,
+            defaults to target_instances. If set, the job will not start until the minimum number of nodes is available.
+        enable_metrics: Whether to enable metrics publishing for the job.
+        database: The database to use.
+        schema: The schema to use.
+        session: The Snowpark session to use. If none specified, uses active session.
+
+
+    Returns:
+        An object representing the submitted job.
+    """
+    return _submit_job(
+        source=source,
+        entrypoint=entrypoint,
+        args=args,
+        compute_pool=compute_pool,
+        stage_name=stage_name,
+        env_vars=env_vars,
+        pip_requirements=pip_requirements,
+        external_access_integrations=external_access_integrations,
+        query_warehouse=query_warehouse,
+        spec_overrides=spec_overrides,
+        target_instances=target_instances,
+        min_instances=min_instances,
+        enable_metrics=enable_metrics,
+        database=database,
+        schema=schema,
+        session=session,
+    )
+
+
 @overload
 def _submit_job(
     source: str,
@@ -235,7 +305,7 @@ def _submit_job(
     query_warehouse: Optional[str] = None,
     spec_overrides: Optional[dict[str, Any]] = None,
     target_instances: int = 1,
-    min_instances: int = 1,
+    min_instances: Optional[int] = None,
     enable_metrics: bool = False,
     database: Optional[str] = None,
     schema: Optional[str] = None,
@@ -258,7 +328,7 @@ def _submit_job(
     query_warehouse: Optional[str] = None,
     spec_overrides: Optional[dict[str, Any]] = None,
     target_instances: int = 1,
-    min_instances: int = 1,
+    min_instances: Optional[int] = None,
     enable_metrics: bool = False,
     database: Optional[str] = None,
     schema: Optional[str] = None,
@@ -292,7 +362,7 @@ def _submit_job(
     query_warehouse: Optional[str] = None,
     spec_overrides: Optional[dict[str, Any]] = None,
     target_instances: int = 1,
-    min_instances: int = 1,
+    min_instances: Optional[int] = None,
     enable_metrics: bool = False,
     database: Optional[str] = None,
     schema: Optional[str] = None,
@@ -313,7 +383,8 @@ def _submit_job(
         query_warehouse: The query warehouse to use. Defaults to session warehouse.
         spec_overrides: Custom service specification overrides to apply.
         target_instances: The number of instances to use for the job. If none specified, single node job is created.
-        min_instances: The minimum number of nodes required to start the job. If none specified, defaults to 1.
+        min_instances: The minimum number of nodes required to start the job. If none specified,
+            defaults to target_instances. If set, the job will not start until the minimum number of nodes is available.
         enable_metrics: Whether to enable metrics publishing for the job.
         database: The database to use.
         schema: The schema to use.
@@ -328,13 +399,25 @@ def _submit_job(
     """
     if database and not schema:
         raise ValueError("Schema must be specified if database is specified.")
-    if target_instances < 1 or min_instances < 1:
-        raise ValueError("target_instances and min_instances must be greater than 0.")
-    if min_instances > target_instances:
-        raise ValueError("min_instances must be less than or equal to target_instances.")
+    if target_instances < 1:
+        raise ValueError("target_instances must be greater than 0.")
+
+    min_instances = target_instances if min_instances is None else min_instances
+    if not (0 < min_instances <= target_instances):
+        raise ValueError("min_instances must be greater than 0 and less than or equal to target_instances.")
 
     session = session or get_active_session()
 
+    if min_instances > 1:
+        # Validate min_instances against compute pool max_nodes
+        pool_info = jb._get_compute_pool_info(session, compute_pool)
+        max_nodes = int(pool_info["max_nodes"])
+        if min_instances > max_nodes:
+            raise ValueError(
+                f"The requested min_instances ({min_instances}) exceeds the max_nodes ({max_nodes}) "
+                f"of compute pool '{compute_pool}'. Reduce min_instances or increase max_nodes."
+            )
+
     # Validate database and schema identifiers on client side since
     # SQL parser for EXECUTE JOB SERVICE seems to struggle with this
     database = identifier.resolve_identifier(cast(str, database or session.get_current_database()))

snowflake/ml/model/_client/model/model_impl.py

@@ -426,3 +426,61 @@ class Model:
             schema_name=new_schema or self._model_ops._model_client._schema_name,
         )
         self._model_name = new_model
+
+    def _repr_html_(self) -> str:
+        """Generate an HTML representation of the model.
+
+        Returns:
+            str: HTML string containing formatted model details.
+        """
+        from snowflake.ml.utils import html_utils
+
+        # Get default version
+        default_version = self.default.version_name
+
+        # Get versions info
+        try:
+            versions_df = self.show_versions()
+            versions_html = ""
+
+            for _, row in versions_df.iterrows():
+                versions_html += html_utils.create_version_item(
+                    version_name=row["name"],
+                    created_on=str(row["created_on"]),
+                    comment=str(row.get("comment", "")),
+                    is_default=bool(row["is_default_version"]),
+                )
+        except Exception:
+            versions_html = html_utils.create_error_message("Error retrieving versions")
+
+        # Get tags
+        try:
+            tags = self.show_tags()
+            if not tags:
+                tags_html = html_utils.create_error_message("No tags available")
+            else:
+                tags_html = ""
+                for tag_name, tag_value in tags.items():
+                    tags_html += html_utils.create_tag_item(tag_name, tag_value)
+        except Exception:
+            tags_html = html_utils.create_error_message("Error retrieving tags")
+
+        # Create main content sections
+        main_info = html_utils.create_grid_section(
+            [
+                ("Model Name", self.name),
+                ("Full Name", self.fully_qualified_name),
+                ("Description", self.description),
+                ("Default Version", default_version),
+            ]
+        )
+
+        versions_section = html_utils.create_section_header("Versions") + html_utils.create_content_section(
+            versions_html
+        )
+
+        tags_section = html_utils.create_section_header("Tags") + html_utils.create_content_section(tags_html)
+
+        content = main_info + versions_section + tags_section
+
+        return html_utils.create_base_container("Model Details", content)

snowflake/ml/model/_client/model/model_version_impl.py

@@ -38,6 +38,96 @@ class ModelVersion(lineage_node.LineageNode):
     def __init__(self) -> None:
         raise RuntimeError("ModelVersion's initializer is not meant to be used. Use `version` from model instead.")
 
+    def _repr_html_(self) -> str:
+        """Generate an HTML representation of the model version.
+
+        Returns:
+            str: HTML string containing formatted model version details.
+        """
+        from snowflake.ml.utils import html_utils
+
+        # Get task
+        try:
+            task = self.get_model_task().value
+        except Exception:
+            task = (
+                html_utils.create_error_message("Not available")
+                .replace('<em style="color: #888; font-style: italic;">', "")
+                .replace("</em>", "")
+            )
+
+        # Get functions info for display
+        try:
+            functions = self.show_functions()
+            if not functions:
+                functions_html = html_utils.create_error_message("No functions available")
+            else:
+                functions_list = []
+                for func in functions:
+                    try:
+                        sig_html = func["signature"]._repr_html_()
+                    except Exception:
+                        # Fallback to simple display if can't display signature
+                        sig_html = f"<pre style='margin: 5px 0;'>{func['signature']}</pre>"
+
+                    function_content = f"""
+                        <div style="margin: 5px 0;">
+                            <strong>Target Method:</strong> {func['target_method']}
+                        </div>
+                        <div style="margin: 5px 0;">
+                            <strong>Function Type:</strong> {func.get('target_method_function_type', 'N/A')}
+                        </div>
+                        <div style="margin: 5px 0;">
+                            <strong>Partitioned:</strong> {func.get('is_partitioned', False)}
+                        </div>
+                        <div style="margin: 10px 0;">
+                            <strong>Signature:</strong>
+                            {sig_html}
+                        </div>
+                    """
+
+                    functions_list.append(
+                        html_utils.create_collapsible_section(
+                            title=func["name"], content=function_content, open_by_default=False
+                        )
+                    )
+                functions_html = "".join(functions_list)
+        except Exception:
+            functions_html = html_utils.create_error_message("Error retrieving functions")
+
+        # Get metrics for display
+        try:
+            metrics = self.show_metrics()
+            if not metrics:
+                metrics_html = html_utils.create_error_message("No metrics available")
+            else:
+                metrics_html = ""
+                for metric_name, value in metrics.items():
+                    metrics_html += html_utils.create_metric_item(metric_name, value)
+        except Exception:
+            metrics_html = html_utils.create_error_message("Error retrieving metrics")
+
+        # Create main content sections
+        main_info = html_utils.create_grid_section(
+            [
+                ("Model Name", self.model_name),
+                ("Version", f'<strong style="color: #28a745;">{self.version_name}</strong>'),
+                ("Full Name", self.fully_qualified_model_name),
+                ("Description", self.description),
+                ("Task", task),
+            ]
+        )
+
+        functions_section = html_utils.create_section_header("Functions") + html_utils.create_content_section(
+            functions_html
+        )
+
+        metrics_section = html_utils.create_section_header("Metrics") + html_utils.create_content_section(metrics_html)
+
+        content = main_info + functions_section + metrics_section
+
+        return html_utils.create_base_container("Model Version Details", content)
+
     @classmethod
     def _ref(
         cls,

snowflake/ml/model/_client/ops/model_ops.py

@@ -643,14 +643,17 @@ class ModelOperator:
         # TODO(sdas): Figure out a better way to filter out MODEL_BUILD_ services server side.
         fully_qualified_service_names = [str(service) for service in json_array if "MODEL_BUILD_" not in service]
 
-        result = []
-
+        result: list[ServiceInfo] = []
         for fully_qualified_service_name in fully_qualified_service_names:
             ingress_url: Optional[str] = None
             db, schema, service_name = sql_identifier.parse_fully_qualified_name(fully_qualified_service_name)
-            service_status, _ = self._service_client.get_service_status(
+            statuses = self._service_client.get_service_container_statuses(
                 database_name=db, schema_name=schema, service_name=service_name, statement_params=statement_params
             )
+            if len(statuses) == 0:
+                return result
+
+            service_status = statuses[0].service_status
             for res_row in self._service_client.show_endpoints(
                 database_name=db, schema_name=schema, service_name=service_name, statement_params=statement_params
             ):