snowflake-ml-python 1.9.0__py3-none-any.whl → 1.9.2__py3-none-any.whl

snowflake/ml/jobs/_utils/spec_utils.py

@@ -12,12 +12,14 @@ from snowflake.ml.jobs._utils import constants, query_helper, types
 def _get_node_resources(session: snowpark.Session, compute_pool: str) -> types.ComputeResources:
     """Extract resource information for the specified compute pool"""
     # Get the instance family
-    rows = session._conn.run_query("show compute pools like ?", params=[compute_pool], _force_qmark_paramstyle=True)
-    if not rows or not isinstance(rows, dict) or not rows.get("data"):
+    rows = query_helper.run_query(
+        session,
+        "show compute pools like ?",
+        params=[compute_pool],
+    )
+    if not rows:
         raise ValueError(f"Compute pool '{compute_pool}' not found")
-    requested_attributes = query_helper.get_attribute_map(session, {"instance_family": 4})
-    compute_pool_info = rows["data"]
-    instance_family: str = compute_pool_info[0][requested_attributes["instance_family"]]
+    instance_family: str = rows[0]["instance_family"]
     cloud = snowflake_env.get_current_cloud(session, default=snowflake_env.SnowflakeCloudType.AWS)
 
     return (
@@ -179,10 +181,10 @@ def generate_service_spec(
     # TODO: Add hooks for endpoints for integration with TensorBoard etc
 
     env_vars = {
-        constants.PAYLOAD_DIR_ENV_VAR: stage_mount.as_posix(),
+        constants.PAYLOAD_DIR_ENV_VAR: constants.APP_MOUNT_PATH,
         constants.RESULT_PATH_ENV_VAR: constants.RESULT_PATH_DEFAULT_VALUE,
     }
-    endpoints = []
+    endpoints: list[dict[str, Any]] = []
 
     if target_instances > 1:
         # Update environment variables for multi-node job
@@ -191,7 +193,7 @@ def generate_service_spec(
         env_vars[constants.MIN_INSTANCES_ENV_VAR] = str(min_instances)
 
         # Define Ray endpoints for intra-service instance communication
-        ray_endpoints = [
+        ray_endpoints: list[dict[str, Any]] = [
             {"name": "ray-client-server-endpoint", "port": 10001, "protocol": "TCP"},
             {"name": "ray-gcs-endpoint", "port": 12001, "protocol": "TCP"},
             {"name": "ray-dashboard-grpc-endpoint", "port": 12002, "protocol": "TCP"},
@@ -234,6 +236,19 @@ def generate_service_spec(
         ],
         "volumes": volumes,
     }
+
+    if target_instances > 1:
+        spec_dict.update(
+            {
+                "resourceManagement": {
+                    "controlPolicy": {
+                        "startupOrder": {
+                            "type": "FirstInstance",
+                        },
+                    },
+                },
+            }
+        )
     if endpoints:
         spec_dict["endpoints"] = endpoints
     if metrics:

snowflake/ml/jobs/_utils/stage_utils.py

@@ -14,7 +14,10 @@ _SNOWURL_PATH_RE = re.compile(
     r"(?P<path>versions(?:/(?P<version>[^/]+)(?:/(?P<relpath>.*))?)?)$"
 )
 
-_STAGEF_PATH_RE = re.compile(r"^@(?P<stage>~|%?\w+)(?:/(?P<relpath>[\w\-./]*))?$")
+# Break long regex into two main parts
+_STAGE_PATTERN = rf"~|%?(?:(?:{identifier._SF_IDENTIFIER}\.?){{,2}}{identifier._SF_IDENTIFIER})"
+_RELPATH_PATTERN = r"[\w\-./]*"
+_STAGEF_PATH_RE = re.compile(rf"^@(?P<stage>{_STAGE_PATTERN})(?:/(?P<relpath>{_RELPATH_PATTERN}))?$")
 
 
 class StagePath:
@@ -29,6 +32,14 @@ class StagePath:
         self._root = self._raw_path[0:start].rstrip("/") if relpath else self._raw_path.rstrip("/")
         self._path = Path(relpath or "")
 
+    @property
+    def parts(self) -> tuple[str, ...]:
+        return self._path.parts
+
+    @property
+    def name(self) -> str:
+        return self._path.name
+
     @property
     def parent(self) -> "StagePath":
         if self._path.parent == Path(""):
@@ -51,18 +62,28 @@ class StagePath:
         else:
             return f"{self.root}/{path}"
 
-    def is_relative_to(self, path: Union[str, PathLike[str], "StagePath"]) -> bool:
+    def is_relative_to(self, *other: Union[str, os.PathLike[str]]) -> bool:
+        if not other:
+            raise TypeError("is_relative_to() requires at least one argument")
+        # For now, we only support a single argument, like pathlib.Path in Python < 3.12
+        path = other[0]
         stage_path = path if isinstance(path, StagePath) else StagePath(os.fspath(path))
         if stage_path.root == self.root:
             return self._path.is_relative_to(stage_path._path)
         else:
             return False
 
-    def relative_to(self, path: Union[str, PathLike[str], "StagePath"]) -> PurePath:
+    def relative_to(self, *other: Union[str, os.PathLike[str]]) -> PurePath:
+        if not other:
+            raise TypeError("relative_to() requires at least one argument")
+        if not self.is_relative_to(*other):
+            raise ValueError(f"{other} does not start with {self._raw_path}")
+        path = other[0]
         stage_path = path if isinstance(path, StagePath) else StagePath(os.fspath(path))
         if self.root == stage_path.root:
             return self._path.relative_to(stage_path._path)
-        raise ValueError(f"{self._raw_path} does not start with {stage_path._raw_path}")
+        else:
+            raise ValueError(f"{self._raw_path} does not start with {stage_path._raw_path}")
 
     def absolute(self) -> "StagePath":
         return self
@@ -88,6 +109,9 @@ class StagePath:
     def __str__(self) -> str:
         return self.as_posix()
 
+    def __repr__(self) -> str:
+        return f"StagePath('{self.as_posix()}')"
+
     def __eq__(self, other: object) -> bool:
         if not isinstance(other, StagePath):
             raise NotImplementedError
@@ -96,24 +120,16 @@ class StagePath:
     def __fspath__(self) -> str:
         return self._compose_path(self._path)
 
-    def joinpath(self, *args: Union[str, PathLike[str], "StagePath"]) -> "StagePath":
+    def joinpath(self, *args: Union[str, PathLike[str]]) -> "StagePath":
         path = self
         for arg in args:
             path = path._make_child(arg)
         return path
 
-    def _make_child(self, path: Union[str, PathLike[str], "StagePath"]) -> "StagePath":
+    def _make_child(self, path: Union[str, PathLike[str]]) -> "StagePath":
         stage_path = path if isinstance(path, StagePath) else StagePath(os.fspath(path))
         if self.root == stage_path.root:
             child_path = self._path.joinpath(stage_path._path)
             return StagePath(self._compose_path(child_path))
         else:
             return stage_path
-
-
-def identify_stage_path(path: str) -> Union[StagePath, Path]:
-    try:
-        stage_path = StagePath(path)
-    except ValueError:
-        return Path(path)
-    return stage_path

snowflake/ml/jobs/_utils/types.py

@@ -1,8 +1,7 @@
+import os
 from dataclasses import dataclass
 from pathlib import PurePath
-from typing import Literal, Optional, Union
-
-from snowflake.ml.jobs._utils import stage_utils
+from typing import Iterator, Literal, Optional, Protocol, Union, runtime_checkable
 
 JOB_STATUS = Literal[
     "PENDING",
@@ -15,9 +14,70 @@ JOB_STATUS = Literal[
 ]
 
 
+@runtime_checkable
+class PayloadPath(Protocol):
+    """A protocol for path-like objects used in this module, covering methods from pathlib.Path and StagePath."""
+
+    @property
+    def name(self) -> str:
+        ...
+
+    @property
+    def suffix(self) -> str:
+        ...
+
+    @property
+    def parent(self) -> "PayloadPath":
+        ...
+
+    def exists(self) -> bool:
+        ...
+
+    def is_file(self) -> bool:
+        ...
+
+    def is_absolute(self) -> bool:
+        ...
+
+    def absolute(self) -> "PayloadPath":
+        ...
+
+    def joinpath(self, *other: Union[str, os.PathLike[str]]) -> "PayloadPath":
+        ...
+
+    def as_posix(self) -> str:
+        ...
+
+    def is_relative_to(self, *other: Union[str, os.PathLike[str]]) -> bool:
+        ...
+
+    def relative_to(self, *other: Union[str, os.PathLike[str]]) -> PurePath:
+        ...
+
+    def __fspath__(self) -> str:
+        ...
+
+    def __str__(self) -> str:
+        ...
+
+    def __repr__(self) -> str:
+        ...
+
+
+@dataclass
+class PayloadSpec:
+    """Represents a payload item to be uploaded."""
+
+    source_path: PayloadPath
+    remote_relative_path: Optional[PurePath] = None
+
+    def __iter__(self) -> Iterator[Union[PayloadPath, Optional[PurePath]]]:
+        return iter((self.source_path, self.remote_relative_path))
+
+
 @dataclass(frozen=True)
 class PayloadEntrypoint:
-    file_path: Union[PurePath, stage_utils.StagePath]
+    file_path: PayloadPath
     main_func: Optional[str]
 
 

snowflake/ml/jobs/job.py

@@ -3,12 +3,12 @@ import logging
 import os
 import time
 from functools import cached_property
+from pathlib import Path
 from typing import Any, Generic, Literal, Optional, TypeVar, Union, cast, overload
 
 import yaml
 
 from snowflake import snowpark
-from snowflake.connector import errors
 from snowflake.ml._internal import telemetry
 from snowflake.ml._internal.utils import identifier
 from snowflake.ml._internal.utils.mixins import SerializableSessionMixin
@@ -70,8 +70,7 @@ class MLJob(Generic[T], SerializableSessionMixin):
     def _compute_pool(self) -> str:
         """Get the job's compute pool name."""
         row = _get_service_info(self._session, self.id)
-        compute_pool = row[query_helper.get_attribute_map(self._session, {"compute_pool": 5})["compute_pool"]]
-        return cast(str, compute_pool)
+        return cast(str, row["compute_pool"])
 
     @property
     def _service_spec(self) -> dict[str, Any]:
@@ -97,10 +96,24 @@ class MLJob(Generic[T], SerializableSessionMixin):
     @property
     def _result_path(self) -> str:
         """Get the job's result file location."""
-        result_path = self._container_spec["env"].get(constants.RESULT_PATH_ENV_VAR)
-        if result_path is None:
+        result_path_str = self._container_spec["env"].get(constants.RESULT_PATH_ENV_VAR)
+        if result_path_str is None:
             raise RuntimeError(f"Job {self.name} doesn't have a result path configured")
-        return f"{self._stage_path}/{result_path}"
+        volume_mounts = self._container_spec["volumeMounts"]
+        stage_mount_str = next(v for v in volume_mounts if v.get("name") == constants.STAGE_VOLUME_NAME)["mountPath"]
+
+        result_path = Path(result_path_str)
+        stage_mount = Path(stage_mount_str)
+        try:
+            relative_path = result_path.relative_to(stage_mount)
+        except ValueError:
+            if result_path.is_absolute():
+                raise ValueError(
+                    f"Result path {result_path} is absolute, but should be relative to stage mount {stage_mount}"
+                )
+            relative_path = result_path
+
+        return f"{self._stage_path}/{relative_path.as_posix()}"
 
     @overload
     def get_logs(
@@ -183,20 +196,17 @@ class MLJob(Generic[T], SerializableSessionMixin):
         start_time = time.monotonic()
         warning_shown = False
         while (status := self.status) not in TERMINAL_JOB_STATUSES:
-            if status == "PENDING" and not warning_shown:
+            elapsed = time.monotonic() - start_time
+            if elapsed >= timeout >= 0:
+                raise TimeoutError(f"Job {self.name} did not complete within {timeout} seconds")
+            elif status == "PENDING" and not warning_shown and elapsed >= 2:  # Only show warning after 2s
                 pool_info = _get_compute_pool_info(self._session, self._compute_pool)
-                requested_attributes = {"max_nodes": 3, "active_nodes": 9}
-                if (
-                    pool_info[requested_attributes["max_nodes"]] - pool_info[requested_attributes["active_nodes"]]
-                ) < self.min_instances:
+                if (pool_info.max_nodes - pool_info.active_nodes) < self.min_instances:
                     logger.warning(
-                        f'Compute pool busy ({pool_info[requested_attributes["active_nodes"]]}'
-                        f'/{pool_info[requested_attributes["max_nodes"]]} nodes in use, '
+                        f"Compute pool busy ({pool_info.active_nodes}/{pool_info.max_nodes} nodes in use, "
                         f"{self.min_instances} nodes required). Job execution may be delayed."
                     )
                     warning_shown = True
-            if timeout >= 0 and (elapsed := time.monotonic() - start_time) >= timeout:
-                raise TimeoutError(f"Job {self.name} did not complete within {elapsed} seconds")
             time.sleep(delay)
             delay = min(delay * 1.2, constants.JOB_POLL_MAX_DELAY_SECONDS)  # Exponential backoff
         return self.status
@@ -247,27 +257,21 @@ def _get_status(session: snowpark.Session, job_id: str, instance_id: Optional[in
     """Retrieve job or job instance execution status."""
     if instance_id is not None:
         # Get specific instance status
-        rows = session._conn.run_query(
-            "SHOW SERVICE INSTANCES IN SERVICE IDENTIFIER(?)", params=[job_id], _force_qmark_paramstyle=True
-        )
-        request_attributes = query_helper.get_attribute_map(session, {"status": 5, "instance_id": 4})
-        if isinstance(rows, dict) and "data" in rows:
-            for row in rows["data"]:
-                if row[request_attributes["instance_id"]] == str(instance_id):
-                    return cast(types.JOB_STATUS, row[request_attributes["status"]])
+        rows = session.sql("SHOW SERVICE INSTANCES IN SERVICE IDENTIFIER(?)", params=(job_id,)).collect()
+        for row in rows:
+            if row["instance_id"] == str(instance_id):
+                return cast(types.JOB_STATUS, row["status"])
         raise ValueError(f"Instance {instance_id} not found in job {job_id}")
     else:
         row = _get_service_info(session, job_id)
-        request_attributes = query_helper.get_attribute_map(session, {"status": 1})
-        return cast(types.JOB_STATUS, row[request_attributes["status"]])
+        return cast(types.JOB_STATUS, row["status"])
 
 
 @telemetry.send_api_usage_telemetry(project=_PROJECT, func_params_to_log=["job_id"])
 def _get_service_spec(session: snowpark.Session, job_id: str) -> dict[str, Any]:
     """Retrieve job execution service spec."""
     row = _get_service_info(session, job_id)
-    requested_attributes = query_helper.get_attribute_map(session, {"spec": 6})
-    return cast(dict[str, Any], yaml.safe_load(row[requested_attributes["spec"]]))
+    return cast(dict[str, Any], yaml.safe_load(row["spec"]))
 
 
 @telemetry.send_api_usage_telemetry(project=_PROJECT, func_params_to_log=["job_id", "limit", "instance_id"])
@@ -307,18 +311,14 @@ def _get_logs(
     if limit > 0:
         params.append(limit)
     try:
-        data = session._conn.run_query(
+        (row,) = query_helper.run_query(
+            session,
             f"SELECT SYSTEM$GET_SERVICE_LOGS(?, ?, ?{f', ?' if limit > 0 else ''})",
             params=params,
-            _force_qmark_paramstyle=True,
         )
-        if isinstance(data, dict) and "data" in data:
-            full_log = str(data["data"][0][0])
-        # pass type check
-        else:
-            full_log = ""
-    except errors.ProgrammingError as e:
-        if "Container Status: PENDING" in str(e):
+        full_log = str(row[0])
+    except SnowparkSQLException as e:
+        if "Container Status: PENDING" in e.message:
             logger.warning("Waiting for container to start. Logs will be shown when available.")
             return ""
         else:
@@ -399,7 +399,7 @@ def _get_head_instance_id(session: snowpark.Session, job_id: str) -> Optional[in
 
     try:
         target_instances = _get_target_instances(session, job_id)
-    except errors.ProgrammingError:
+    except SnowparkSQLException:
         # service may be deleted
         raise RuntimeError("Couldn’t retrieve service information")
 
@@ -407,34 +407,32 @@ def _get_head_instance_id(session: snowpark.Session, job_id: str) -> Optional[in
         return 0
 
     try:
-        rows = session._conn.run_query(
-            "SHOW SERVICE INSTANCES IN SERVICE IDENTIFIER(?)", params=(job_id,), _force_qmark_paramstyle=True
+        rows = query_helper.run_query(
+            session,
+            "SHOW SERVICE INSTANCES IN SERVICE IDENTIFIER(?)",
+            params=(job_id,),
         )
-    except errors.ProgrammingError:
+    except SnowparkSQLException:
         # service may be deleted
         raise RuntimeError("Couldn’t retrieve instances")
 
-    if not rows or not isinstance(rows, dict) or not rows.get("data"):
+    if not rows:
         return None
 
-    if target_instances > len(rows["data"]):
+    if target_instances > len(rows):
         raise RuntimeError("Couldn’t retrieve head instance due to missing instances.")
 
-    requested_attributes = query_helper.get_attribute_map(session, {"start_time": 8, "instance_id": 4})
     # Sort by start_time first, then by instance_id
     try:
-        sorted_instances = sorted(
-            rows["data"],
-            key=lambda x: (x[requested_attributes["start_time"]], int(x[requested_attributes["instance_id"]])),
-        )
+        sorted_instances = sorted(rows, key=lambda x: (x["start_time"], int(x["instance_id"])))
     except TypeError:
         raise RuntimeError("Job instance information unavailable.")
     head_instance = sorted_instances[0]
-    if not head_instance[requested_attributes["start_time"]]:
+    if not head_instance["start_time"]:
         # If head instance hasn't started yet, return None
         return None
     try:
-        return int(head_instance[requested_attributes["instance_id"]])
+        return int(head_instance["instance_id"])
     except (ValueError, TypeError):
         return 0
 
@@ -446,14 +444,16 @@ def _get_service_log_from_event_table(
     schema: Optional[str] = None,
     instance_id: Optional[int] = None,
     limit: int = -1,
-) -> Any:
-    params: list[Any] = [
-        name,
-    ]
+) -> list[Row]:
+    event_table_name = session.sql("SHOW PARAMETERS LIKE 'event_table' IN ACCOUNT").collect()[0]["value"]
     query = [
-        "SELECT VALUE FROM snowflake.telemetry.events_view",
+        "SELECT VALUE FROM IDENTIFIER(?)",
         'WHERE RESOURCE_ATTRIBUTES:"snow.service.name" = ?',
     ]
+    params: list[Any] = [
+        event_table_name,
+        name,
+    ]
     if database:
         query.append('AND RESOURCE_ATTRIBUTES:"snow.database.name" = ?')
         params.append(database)
@@ -473,23 +473,22 @@ def _get_service_log_from_event_table(
     if limit > 0:
         query.append("LIMIT ?")
         params.append(limit)
-    rows = session._conn.run_query(
-        "\n".join(line for line in query if line), params=params, _force_qmark_paramstyle=True
+    # the wrap used in query_helper does not have return type.
+    # sticking a # type: ignore[no-any-return] is to pass type check
+    rows = query_helper.run_query(
+        session,
+        "\n".join(line for line in query if line),
+        params=params,
     )
-    if not rows or not isinstance(rows, dict) or not rows.get("data"):
-        return []
-    return rows["data"]
+    return rows  # type: ignore[no-any-return]
 
 
 def _get_service_info(session: snowpark.Session, job_id: str) -> Any:
-    row = session._conn.run_query("DESCRIBE SERVICE IDENTIFIER(?)", params=(job_id,), _force_qmark_paramstyle=True)
-    # pass the type check
-    if not row or not isinstance(row, dict) or not row.get("data"):
-        raise errors.ProgrammingError("failed to retrieve service information")
-    return row["data"][0]
+    (row,) = query_helper.run_query(session, "DESCRIBE SERVICE IDENTIFIER(?)", params=(job_id,))
+    return row
 
 
-def _get_compute_pool_info(session: snowpark.Session, compute_pool: str) -> Any:
+def _get_compute_pool_info(session: snowpark.Session, compute_pool: str) -> Row:
     """
     Check if the compute pool has enough available instances.
 
@@ -498,19 +497,16 @@ def _get_compute_pool_info(session: snowpark.Session, compute_pool: str) -> Any:
         compute_pool (str): The name of the compute pool.
 
     Returns:
-        Any: The compute pool information.
+        Row: The compute pool information.
 
     Raises:
         ValueError: If the compute pool is not found.
     """
     try:
-        compute_pool_info = session._conn.run_query(
-            "SHOW COMPUTE POOLS LIKE ?", params=(compute_pool,), _force_qmark_paramstyle=True
-        )
-        # pass the type check
-        if not compute_pool_info or not isinstance(compute_pool_info, dict) or not compute_pool_info.get("data"):
-            raise ValueError(f"Compute pool '{compute_pool}' not found")
-        return compute_pool_info["data"][0]
+        # the wrap used in query_helper does not have return type.
+        # sticking a # type: ignore[no-any-return] is to pass type check
+        (pool_info,) = query_helper.run_query(session, "SHOW COMPUTE POOLS LIKE ?", params=(compute_pool,))
+        return pool_info  # type: ignore[no-any-return]
     except ValueError as e:
         if "not enough values to unpack" in str(e):
             raise ValueError(f"Compute pool '{compute_pool}' not found")
@@ -520,8 +516,7 @@ def _get_compute_pool_info(session: snowpark.Session, compute_pool: str) -> Any:
 @telemetry.send_api_usage_telemetry(project=_PROJECT, func_params_to_log=["job_id"])
 def _get_target_instances(session: snowpark.Session, job_id: str) -> int:
     row = _get_service_info(session, job_id)
-    requested_attributes = query_helper.get_attribute_map(session, {"target_instances": 9})
-    return int(row[requested_attributes["target_instances"]])
+    return int(row["target_instances"])
 
 
 def _get_logs_spcs(

snowflake/ml/jobs/manager.py

@@ -8,7 +8,6 @@ import pandas as pd
 import yaml
 
 from snowflake import snowpark
-from snowflake.connector import errors
 from snowflake.ml._internal import telemetry
 from snowflake.ml._internal.utils import identifier
 from snowflake.ml.jobs import job as jb
@@ -169,8 +168,8 @@ def get_job(job_id: str, session: Optional[snowpark.Session] = None) -> jb.MLJob
         job = jb.MLJob[Any](job_id, session=session)
         _ = job._service_spec
         return job
-    except errors.ProgrammingError as e:
-        if "does not exist" in str(e):
+    except SnowparkSQLException as e:
+        if "does not exist" in e.message:
             raise ValueError(f"Job does not exist: {job_id}") from e
         raise
 
@@ -186,7 +185,7 @@ def delete_job(job: Union[str, jb.MLJob[Any]], session: Optional[snowpark.Sessio
         logger.info(f"Successfully cleaned up stage files for job {job.id} at {stage_path}")
     except Exception as e:
         logger.warning(f"Failed to clean up stage files for job {job.id}: {e}")
-    session._conn.run_query("DROP SERVICE IDENTIFIER(?)", params=(job.id,), _force_qmark_paramstyle=True)
+    query_helper.run_query(session, "DROP SERVICE IDENTIFIER(?)", params=(job.id,))
 
 
 @telemetry.send_api_usage_telemetry(project=_PROJECT)
@@ -426,12 +425,18 @@ def _submit_job(
         An object representing the submitted job.
 
     Raises:
-        RuntimeError: If required Snowflake features are not enabled.
         ValueError: If database or schema value(s) are invalid
-        errors.ProgrammingError: if the SQL query or its parameters are invalid
+        SnowparkSQLException: If there is an error submitting the job.
     """
     session = session or get_active_session()
 
+    # Check for deprecated args
+    if "num_instances" in kwargs:
+        logger.warning(
+            "'num_instances' is deprecated and will be removed in a future release. Use 'target_instances' instead."
+        )
+        target_instances = max(target_instances, kwargs.pop("num_instances"))
+
     # Use kwargs for less common optional parameters
     database = kwargs.pop("database", None)
     schema = kwargs.pop("schema", None)
@@ -442,13 +447,10 @@ def _submit_job(
     spec_overrides = kwargs.pop("spec_overrides", None)
     enable_metrics = kwargs.pop("enable_metrics", True)
     query_warehouse = kwargs.pop("query_warehouse", None)
+    additional_payloads = kwargs.pop("additional_payloads", None)
 
-    # Check for deprecated args
-    if "num_instances" in kwargs:
-        logger.warning(
-            "'num_instances' is deprecated and will be removed in a future release. Use 'target_instances' instead."
-        )
-        target_instances = max(target_instances, kwargs.pop("num_instances"))
+    if additional_payloads:
+        logger.warning("'additional_payloads' is in private preview since 1.9.1. Do not use it in production.")
 
     # Warn if there are unknown kwargs
     if kwargs:
@@ -464,8 +466,7 @@ def _submit_job(
     if min_instances > 1:
         # Validate min_instances against compute pool max_nodes
         pool_info = jb._get_compute_pool_info(session, compute_pool)
-        requested_attributes = query_helper.get_attribute_map(session, {"max_nodes": 3})
-        max_nodes = int(pool_info[requested_attributes["max_nodes"]])
+        max_nodes = int(pool_info["max_nodes"])
         if min_instances > max_nodes:
             raise ValueError(
                 f"The requested min_instances ({min_instances}) exceeds the max_nodes ({max_nodes}) "
@@ -480,9 +481,7 @@ def _submit_job(
 
     # Upload payload
     uploaded_payload = payload_utils.JobPayload(
-        source,
-        entrypoint=entrypoint,
-        pip_requirements=pip_requirements,
+        source, entrypoint=entrypoint, pip_requirements=pip_requirements, additional_payloads=additional_payloads
     ).upload(session, stage_path)
 
     # Generate service spec
@@ -502,7 +501,48 @@ def _submit_job(
     if spec_overrides:
         spec = spec_utils.merge_patch(spec, spec_overrides, display_name="spec_overrides")
 
-    # Generate SQL command for job submission
+    query_text, params = _generate_submission_query(
+        spec, external_access_integrations, query_warehouse, target_instances, session, compute_pool, job_id
+    )
+    try:
+        _ = query_helper.run_query(session, query_text, params=params)
+    except SnowparkSQLException as e:
+        if "Invalid spec: unknown option 'resourceManagement' for 'spec'." in e.message:
+            logger.warning("Dropping 'resourceManagement' from spec because control policy is not enabled.")
+            spec["spec"].pop("resourceManagement", None)
+            query_text, params = _generate_submission_query(
+                spec, external_access_integrations, query_warehouse, target_instances, session, compute_pool, job_id
+            )
+            _ = query_helper.run_query(session, query_text, params=params)
+        else:
+            raise
+    return get_job(job_id, session=session)
+
+
+def _generate_submission_query(
+    spec: dict[str, Any],
+    external_access_integrations: list[str],
+    query_warehouse: Optional[str],
+    target_instances: int,
+    session: snowpark.Session,
+    compute_pool: str,
+    job_id: str,
+) -> tuple[str, list[Any]]:
+    """
+    Generate the SQL query for job submission.
+
+    Args:
+        spec: The service spec for the job.
+        external_access_integrations: The external access integrations for the job.
+        query_warehouse: The query warehouse for the job.
+        target_instances: The number of instances for the job.
+        session: The Snowpark session to use.
+        compute_pool: The compute pool to use for the job.
+        job_id: The ID of the job.
+
+    Returns:
+        A tuple containing the SQL query text and the parameters for the query.
+    """
     query_template = textwrap.dedent(
         """\
         EXECUTE JOB SERVICE
@@ -526,17 +566,5 @@ def _submit_job(
     if target_instances > 1:
         query.append("REPLICAS = ?")
         params.append(target_instances)
-
-    # Submit job
     query_text = "\n".join(line for line in query if line)
-
-    try:
-        _ = session._conn.run_query(query_text, params=params, _force_qmark_paramstyle=True)
-    except errors.ProgrammingError as e:
-        if "invalid property 'ASYNC'" in str(e):
-            raise RuntimeError(
-                "SPCS Async Jobs not enabled. Set parameter `ENABLE_SNOWSERVICES_ASYNC_JOBS = TRUE` to enable."
-            ) from e
-        raise
-
-    return get_job(job_id, session=session)
+    return query_text, params