snowflake-ml-python 1.21.0__py3-none-any.whl → 1.23.0__py3-none-any.whl

snowflake/ml/_internal/utils/url.py

@@ -0,0 +1,42 @@
+from urllib.parse import urlunparse
+
+from snowflake import snowpark as snowpark
+
+JOB_URL_PREFIX = "#/compute/job/"
+SERVICE_URL_PREFIX = "#/compute/service/"
+
+
+def get_snowflake_url(
+    session: snowpark.Session,
+    url_path: str,
+    params: str = "",
+    query: str = "",
+    fragment: str = "",
+) -> str:
+    """Construct a Snowflake URL from session connection details and URL components.
+
+    Args:
+        session: The Snowpark session containing connection details.
+        url_path: The path component of the URL (e.g., "/compute/job/123").
+        params: Optional parameters for the URL (RFC 1808). Defaults to "".
+        query: Optional query string for the URL. Defaults to "".
+        fragment: Optional fragment identifier for the URL (e.g., "#section"). Defaults to "".
+
+    Returns:
+        A fully constructed Snowflake URL string with scheme, host, and specified components.
+    """
+    scheme = "https"
+    if hasattr(session.connection, "scheme"):
+        scheme = session.connection.scheme
+    host = session.connection.host
+
+    return urlunparse(
+        (
+            scheme,
+            host,
+            url_path,
+            params,
+            query,
+            fragment,
+        )
+    )

snowflake/ml/jobs/__init__.py

@@ -2,6 +2,7 @@ from snowflake.ml.jobs._interop.exception_utils import install_exception_display
 from snowflake.ml.jobs._utils.types import JOB_STATUS
 from snowflake.ml.jobs.decorators import remote
 from snowflake.ml.jobs.job import MLJob
+from snowflake.ml.jobs.job_definition import MLJobDefinition
 from snowflake.ml.jobs.manager import (
     delete_job,
     get_job,
@@ -24,4 +25,5 @@ __all__ = [
     "MLJob",
     "JOB_STATUS",
     "submit_from_stage",
+    "MLJobDefinition",
 ]

snowflake/ml/jobs/_utils/constants.py

@@ -5,6 +5,7 @@ from snowflake.ml.jobs._utils.types import ComputeResources
 DEFAULT_CONTAINER_NAME = "main"
 MEMORY_VOLUME_NAME = "dshm"
 STAGE_VOLUME_NAME = "stage-volume"
+DEFAULT_PYTHON_VERSION = "3.10"
 
 # Environment variables
 STAGE_MOUNT_PATH_ENV_VAR = "MLRS_STAGE_MOUNT_PATH"
@@ -30,6 +31,7 @@ DEFAULT_IMAGE_CPU = "st_plat/runtime/x86/runtime_image/snowbooks"
 DEFAULT_IMAGE_GPU = "st_plat/runtime/x86/generic_gpu/runtime_image/snowbooks"
 DEFAULT_IMAGE_TAG = "1.8.0"
 DEFAULT_ENTRYPOINT_PATH = "func.py"
+DEFAULT_PYTHON_VERSION = "3.10"
 
 # Percent of container memory to allocate for /dev/shm volume
 MEMORY_VOLUME_SIZE = 0.3

snowflake/ml/jobs/_utils/payload_utils.py

@@ -11,6 +11,7 @@ from importlib.abc import Traversable
 from pathlib import Path, PurePath
 from types import ModuleType
 from typing import IO, Any, Callable, Optional, Union, cast, get_args, get_origin
+from uuid import uuid4
 
 import cloudpickle as cp
 from packaging import version
@@ -36,10 +37,15 @@ _SUPPORTED_ARG_TYPES = {str, int, float}
 _SUPPORTED_ENTRYPOINT_EXTENSIONS = {".py"}
 _ENTRYPOINT_FUNC_NAME = "func"
 _STARTUP_SCRIPT_PATH = PurePath("startup.sh")
+JOB_ID_PREFIX = "MLJOB_"
 
 
 def _compress_and_upload_file(
-    session: snowpark.Session, source_path: Path, stage_path: PurePath, import_path: Optional[str] = None
+    session: snowpark.Session,
+    source_path: Path,
+    stage_path: PurePath,
+    import_path: Optional[str] = None,
+    overwrite: bool = True,
 ) -> None:
     absolute_source_path = source_path.absolute()
     leading_path = absolute_source_path.as_posix()[: -len(import_path)] if import_path else None
@@ -49,11 +55,13 @@ def _compress_and_upload_file(
             cast(IO[bytes], stream),
             stage_path.joinpath(filename).as_posix(),
             auto_compress=False,
-            overwrite=True,
+            overwrite=overwrite,
         )
 
 
-def _upload_directory(session: snowpark.Session, source_path: Path, payload_stage_path: PurePath) -> None:
+def _upload_directory(
+    session: snowpark.Session, source_path: Path, payload_stage_path: PurePath, overwrite: bool = True
+) -> None:
     # Manually traverse the directory and upload each file, since Snowflake PUT
     # can't handle directories. Reduce the number of PUT operations by using
     # wildcard patterns to batch upload files with the same extension.
@@ -81,12 +89,14 @@ def _upload_directory(session: snowpark.Session, source_path: Path, payload_stag
         session.file.put(
             str(path),
             payload_stage_path.joinpath(path.parent.relative_to(source_path)).as_posix(),
-            overwrite=True,
+            overwrite=overwrite,
             auto_compress=False,
         )
 
 
-def upload_payloads(session: snowpark.Session, stage_path: PurePath, *payload_specs: types.PayloadSpec) -> None:
+def upload_payloads(
+    session: snowpark.Session, stage_path: PurePath, *payload_specs: types.PayloadSpec, overwrite: bool = True
+) -> None:
     for spec in payload_specs:
         source_path = spec.source_path
         remote_relative_path = spec.remote_relative_path
@@ -109,6 +119,7 @@ def upload_payloads(session: snowpark.Session, stage_path: PurePath, *payload_sp
                         source_path,
                         stage_path,
                         remote_relative_path.as_posix() if remote_relative_path else None,
+                        overwrite=overwrite,
                     )
                 else:
                     _upload_directory(session, source_path, payload_stage_path)
@@ -120,12 +131,13 @@ def upload_payloads(session: snowpark.Session, stage_path: PurePath, *payload_sp
                         source_path,
                         stage_path,
                         remote_relative_path.as_posix() if remote_relative_path else None,
+                        overwrite=overwrite,
                     )
                 else:
                     session.file.put(
                         str(source_path.resolve()),
                         payload_stage_path.as_posix(),
-                        overwrite=True,
+                        overwrite=overwrite,
                         auto_compress=False,
                     )
 
@@ -455,7 +467,9 @@ class JobPayload:
         self.pip_requirements = pip_requirements
         self.imports = imports
 
-    def upload(self, session: snowpark.Session, stage_path: Union[str, PurePath]) -> types.UploadedPayload:
+    def upload(
+        self, session: snowpark.Session, stage_path: Union[str, PurePath], overwrite: bool = False
+    ) -> types.UploadedPayload:
         # Prepare local variables
         stage_path = PurePath(stage_path) if isinstance(stage_path, str) else stage_path
         source = resolve_source(self.source)
@@ -482,7 +496,6 @@ class JobPayload:
 
         # Handle list entrypoints (custom commands like ["arctic_training"])
         if isinstance(entrypoint, (list, tuple)):
-            payload_name = entrypoint[0] if entrypoint else None
             # For list entrypoints, still upload source if it's a path
             if isinstance(source, Path):
                 upload_payloads(session, app_stage_path, types.PayloadSpec(source, None))
@@ -491,30 +504,24 @@ class JobPayload:
             python_entrypoint: list[Union[str, PurePath]] = list(entrypoint)
         else:
             # Standard file-based entrypoint handling
-            payload_name = None
             if not isinstance(source, types.PayloadPath):
-                if isinstance(source, function_payload_utils.FunctionPayload):
-                    payload_name = source.function.__name__
-
                 source_code = generate_python_code(source, source_code_display=True)
                 _ = session.file.put_stream(
                     io.BytesIO(source_code.encode()),
                     stage_location=app_stage_path.joinpath(entrypoint.file_path).as_posix(),
                     auto_compress=False,
-                    overwrite=True,
+                    overwrite=overwrite,
                 )
                 source = Path(entrypoint.file_path.parent)
 
             elif isinstance(source, stage_utils.StagePath):
-                payload_name = entrypoint.file_path.stem
                 # copy payload to stage
                 if source == entrypoint.file_path:
                     source = source.parent
-                upload_payloads(session, app_stage_path, types.PayloadSpec(source, None))
+                upload_payloads(session, app_stage_path, types.PayloadSpec(source, None), overwrite=overwrite)
 
             elif isinstance(source, Path):
-                payload_name = entrypoint.file_path.stem
-                upload_payloads(session, app_stage_path, types.PayloadSpec(source, None))
+                upload_payloads(session, app_stage_path, types.PayloadSpec(source, None), overwrite=overwrite)
                 if source.is_file():
                     source = source.parent
 
@@ -565,7 +572,6 @@ class JobPayload:
                 *python_entrypoint,
             ],
             env_vars=env_vars,
-            payload_name=payload_name,
         )
 
 
@@ -759,3 +765,17 @@ def create_function_payload(
     payload = function_payload_utils.FunctionPayload(func, session, session_argument, *bound.args, **bound.kwargs)
 
     return payload
+
+
+def get_payload_name(source: Union[str, Callable[..., Any]], entrypoint: Optional[Union[str, list[str]]] = None) -> str:
+
+    if entrypoint and isinstance(entrypoint, (list, tuple)):
+        return entrypoint[0]
+    elif entrypoint and isinstance(entrypoint, str):
+        return f"{PurePath(entrypoint).stem}"
+    elif source and not callable(source):
+        return f"{PurePath(source).stem}"
+    elif isinstance(source, function_payload_utils.FunctionPayload):
+        return f"{source.function.__name__}"
+    else:
+        return f"{JOB_ID_PREFIX}{str(uuid4()).replace('-', '_').upper()}"

snowflake/ml/jobs/_utils/query_helper.py

@@ -14,10 +14,17 @@ def result_set_to_rows(session: snowpark.Session, result: dict[str, Any]) -> lis
 
 
 @snowflake_plan.SnowflakePlan.Decorator.wrap_exception  # type: ignore[misc]
-def run_query(session: snowpark.Session, query_text: str, params: Optional[Sequence[Any]] = None) -> list[Row]:
+def run_query(
+    session: snowpark.Session,
+    query_text: str,
+    params: Optional[Sequence[Any]] = None,
+    statement_params: Optional[dict[str, Any]] = None,
+) -> list[Row]:
     kwargs: dict[str, Any] = {"query": query_text, "params": params}
     if not is_in_stored_procedure():  # type: ignore[no-untyped-call]
         kwargs["_force_qmark_paramstyle"] = True
+    if statement_params:
+        kwargs["_statement_params"] = statement_params
     result = session._conn.run_query(**kwargs)
     if not isinstance(result, dict) or "data" not in result:
         raise ValueError(f"Unprocessable result: {result}")

snowflake/ml/jobs/_utils/runtime_env_utils.py

@@ -1,8 +1,13 @@
-from typing import Any, Optional, Union
+import datetime
+import logging
+from typing import Any, Literal, Optional, Union
 
 from packaging.version import Version
 from pydantic import BaseModel, Field, RootModel, field_validator
 
+from snowflake import snowpark
+from snowflake.ml.jobs._utils import constants, query_helper
+
 
 class SpcsContainerRuntime(BaseModel):
     python_version: Version = Field(alias="pythonVersion")
@@ -27,6 +32,8 @@ class SpcsContainerRuntime(BaseModel):
 
 class RuntimeEnvironmentEntry(BaseModel):
     spcs_container_runtime: Optional[SpcsContainerRuntime] = Field(alias="spcsContainerRuntime", default=None)
+    created_on: datetime.datetime = Field(alias="createdOn")
+    id: Optional[str] = Field(alias="id")
 
     class Config:
         extra = "allow"
@@ -57,7 +64,54 @@ class RuntimeEnvironmentsDict(RootModel[dict[str, RuntimeEnvironmentEntry]]):
         # Filter out any key whose value is not a dict
         return {key: value for key, value in data.items() if isinstance(value, dict)}
 
-    def get_spcs_container_runtimes(self) -> list[SpcsContainerRuntime]:
-        return [
-            entry.spcs_container_runtime for entry in self.root.values() if entry.spcs_container_runtime is not None
+    def get_spcs_container_runtimes(
+        self,
+        *,
+        hardware_type: Optional[str] = None,
+        python_version: Optional[Version] = None,
+    ) -> list[SpcsContainerRuntime]:
+        # TODO(SNOW-2682000): parse version from NRE in a safer way, like relying on the label,id or image tag.
+        entries: list[RuntimeEnvironmentEntry] = [
+            entry
+            for entry in self.root.values()
+            if entry.spcs_container_runtime is not None
+            and (hardware_type is None or entry.spcs_container_runtime.hardware_type.lower() == hardware_type.lower())
+            and (
+                python_version is None
+                or (
+                    entry.spcs_container_runtime.python_version.major == python_version.major
+                    and entry.spcs_container_runtime.python_version.minor == python_version.minor
+                )
+            )
         ]
+        entries.sort(key=lambda e: e.created_on, reverse=True)
+
+        return [entry.spcs_container_runtime for entry in entries if entry.spcs_container_runtime is not None]
+
+
+def _extract_image_tag(image_url: str) -> Optional[str]:
+    image_tag = image_url.rsplit(":", 1)[-1]
+    return image_tag
+
+
+def find_runtime_image(
+    session: snowpark.Session, target_hardware: Literal["CPU", "GPU"], target_python_version: Optional[str] = None
+) -> Optional[str]:
+    python_version = (
+        Version(target_python_version) if target_python_version else Version(constants.DEFAULT_PYTHON_VERSION)
+    )
+    rows = query_helper.run_query(session, "CALL SYSTEM$NOTEBOOKS_FIND_LABELED_RUNTIMES()")
+    if not rows:
+        return None
+    try:
+        runtime_envs = RuntimeEnvironmentsDict.model_validate_json(rows[0][0])
+        spcs_container_runtimes = runtime_envs.get_spcs_container_runtimes(
+            hardware_type=target_hardware,
+            python_version=python_version,
+        )
+    except Exception as e:
+        logging.warning(f"Failed to parse runtime image name from {rows[0][0]}, error: {e}")
+        return None
+
+    selected_runtime = spcs_container_runtimes[0] if spcs_container_runtimes else None
+    return selected_runtime.runtime_container_image if selected_runtime else None

snowflake/ml/jobs/_utils/spec_utils.py

@@ -1,11 +1,6 @@
-import logging
-import sys
-from typing import Literal, Optional
-
 from snowflake import snowpark
 from snowflake.ml._internal.utils import snowflake_env
 from snowflake.ml.jobs._utils import constants, query_helper, types
-from snowflake.ml.jobs._utils.runtime_env_utils import RuntimeEnvironmentsDict
 
 
 def _get_node_resources(session: snowpark.Session, compute_pool: str) -> types.ComputeResources:
@@ -25,29 +20,3 @@ def _get_node_resources(session: snowpark.Session, compute_pool: str) -> types.C
         constants.COMMON_INSTANCE_FAMILIES.get(instance_family)
         or constants.CLOUD_INSTANCE_FAMILIES[cloud][instance_family]
     )
-
-
-def _get_runtime_image(session: snowpark.Session, target_hardware: Literal["CPU", "GPU"]) -> Optional[str]:
-    rows = query_helper.run_query(session, "CALL SYSTEM$NOTEBOOKS_FIND_LABELED_RUNTIMES()")
-    if not rows:
-        return None
-    try:
-        runtime_envs = RuntimeEnvironmentsDict.model_validate_json(rows[0][0])
-        spcs_container_runtimes = runtime_envs.get_spcs_container_runtimes()
-    except Exception as e:
-        logging.warning(f"Failed to parse runtime image name from {rows[0][0]}, error: {e}")
-        return None
-
-    selected_runtime = next(
-        (
-            runtime
-            for runtime in spcs_container_runtimes
-            if (
-                runtime.hardware_type.lower() == target_hardware.lower()
-                and runtime.python_version.major == sys.version_info.major
-                and runtime.python_version.minor == sys.version_info.minor
-            )
-        ),
-        None,
-    )
-    return selected_runtime.runtime_container_image if selected_runtime else None

snowflake/ml/jobs/_utils/stage_utils.py

@@ -52,7 +52,7 @@ class StagePath:
         if self._path.parent == Path(""):
             return StagePath(self._root)
         else:
-            return StagePath(f"{self._root}/{self._path.parent}")
+            return StagePath(f"{self._root}/{self._path.parent.as_posix()}")
 
     @property
     def root(self) -> str:
@@ -67,7 +67,7 @@ class StagePath:
         if path == Path(""):
             return self.root
         else:
-            return f"{self.root}/{path}"
+            return f"{self.root}/{path.as_posix()}"
 
     def is_relative_to(self, *other: Union[str, os.PathLike[str]]) -> bool:
         if not other:

snowflake/ml/jobs/_utils/types.py

@@ -1,7 +1,7 @@
 import os
 from dataclasses import dataclass, field
 from pathlib import PurePath
-from typing import Literal, Optional, Protocol, Union, runtime_checkable
+from typing import Any, Literal, Optional, Protocol, Union, runtime_checkable
 
 from typing_extensions import Self
 
@@ -103,7 +103,6 @@ class UploadedPayload:
     stage_path: PurePath
     entrypoint: list[Union[str, PurePath]]
     env_vars: dict[str, str] = field(default_factory=dict)
-    payload_name: Optional[str] = None
 
 
 @dataclass(frozen=True)
@@ -128,3 +127,24 @@ class ServiceInfo:
     status: str
     compute_pool: str
     target_instances: int
+
+
+@dataclass
+class JobOptions:
+    external_access_integrations: Optional[list[str]] = None
+    query_warehouse: Optional[str] = None
+    target_instances: Optional[int] = None
+    min_instances: Optional[int] = None
+    use_async: Optional[bool] = True
+    generate_suffix: Optional[bool] = True
+
+
+@dataclass
+class SpecOptions:
+    stage_path: str
+    args: Optional[list[str]] = None
+    env_vars: Optional[dict[str, str]] = None
+    enable_metrics: Optional[bool] = None
+    spec_overrides: Optional[dict[str, Any]] = None
+    runtime: Optional[str] = None
+    enable_stage_mount_v2: Optional[bool] = True

snowflake/ml/jobs/job_definition.py

@@ -0,0 +1,232 @@
+import dataclasses
+import json
+import logging
+import os
+import sys
+from pathlib import PurePath, PurePosixPath
+from typing import Any, Callable, Generic, Optional, TypeVar, Union
+from uuid import uuid4
+
+from typing_extensions import ParamSpec
+
+from snowflake import snowpark
+from snowflake.ml._internal import telemetry
+from snowflake.ml._internal.utils import identifier
+from snowflake.ml._internal.utils.mixins import SerializableSessionMixin
+from snowflake.ml.jobs import job as jb
+from snowflake.ml.jobs._utils import (
+    constants,
+    feature_flags,
+    payload_utils,
+    query_helper,
+    types,
+)
+from snowflake.snowpark import context as sp_context
+from snowflake.snowpark.exceptions import SnowparkSQLException
+
+_Args = ParamSpec("_Args")
+_ReturnValue = TypeVar("_ReturnValue")
+JOB_ID_PREFIX = "MLJOB_"
+_PROJECT = "MLJob"
+logger = logging.getLogger(__name__)
+
+
+class MLJobDefinition(Generic[_Args, _ReturnValue], SerializableSessionMixin):
+    def __init__(
+        self,
+        job_options: types.JobOptions,
+        spec_options: types.SpecOptions,
+        stage_name: str,
+        compute_pool: str,
+        name: str,
+        entrypoint_args: list[Any],
+        database: Optional[str] = None,
+        schema: Optional[str] = None,
+        session: Optional[snowpark.Session] = None,
+    ) -> None:
+        self.stage_name = stage_name
+        self.job_options = job_options
+        self.spec_options = spec_options
+        self.compute_pool = compute_pool
+        self.session = session or sp_context.get_active_session()
+        self.database = database or self.session.get_current_database()
+        self.schema = schema or self.session.get_current_schema()
+        self.job_definition_id = identifier.get_schema_level_object_identifier(self.database, self.schema, name)
+        self.entrypoint_args = entrypoint_args
+
+    def delete(self) -> None:
+        if self.stage_name:
+            try:
+                self.session.sql(f"REMOVE {self.stage_name}/").collect()
+                logger.debug(f"Successfully cleaned up stage files for job definition {self.stage_name}")
+            except Exception as e:
+                logger.warning(f"Failed to clean up stage files for job definition {self.stage_name}: {e}")
+
+    def _prepare_arguments(self, *args: _Args.args, **kwargs: _Args.kwargs) -> list[Any]:
+        # TODO: Add ArgProtocol and respective logics
+        return [arg for arg in args]
+
+    @telemetry.send_api_usage_telemetry(project=_PROJECT)
+    def __call__(self, *args: _Args.args, **kwargs: _Args.kwargs) -> jb.MLJob[_ReturnValue]:
+        statement_params = telemetry.get_statement_params(_PROJECT)
+        statement_params = telemetry.add_statement_params_custom_tags(
+            statement_params,
+            custom_tags={
+                "job_definition_id": self.job_definition_id,
+            },
+        )
+        args_list = self._prepare_arguments(*args, **kwargs)
+        query = self.to_sql(job_args=args_list, use_async=True)
+        job_id = query_helper.run_query(self.session, query, statement_params=statement_params)[0][0]
+        return jb.MLJob[_ReturnValue](job_id, session=self.session)
+
+    @telemetry.send_api_usage_telemetry(project=_PROJECT)
+    def to_sql(self, *, job_args: Optional[list[Any]] = None, use_async: bool = False) -> str:
+        # Combine the entrypoint_args and job_args for use in the query
+        combined_args = (self.entrypoint_args or []) + (job_args or [])
+        spec_options = dataclasses.replace(self.spec_options, args=combined_args)
+        # Uppercase option keys to match the expected SYSTEM$EXECUTE_ML_JOB parameter format
+        spec_options_dict = {k.upper(): v for k, v in dataclasses.asdict(spec_options).items()}
+        job_options = dataclasses.replace(self.job_options, use_async=use_async)
+        # Uppercase option keys to match the expected SYSTEM$EXECUTE_ML_JOB parameter format
+        job_options_dict = {k.upper(): v for k, v in dataclasses.asdict(job_options).items()}
+        job_options_dict["ASYNC"] = job_options_dict.pop("USE_ASYNC")
+        params = [
+            self.job_definition_id + ("_" if self.job_options.generate_suffix else ""),
+            self.compute_pool,
+            json.dumps(spec_options_dict),
+            json.dumps(job_options_dict),
+        ]
+        query_template = "CALL SYSTEM$EXECUTE_ML_JOB(%s, %s, %s, %s)"
+        sql = self.session._conn._cursor._preprocess_pyformat_query(query_template, params)
+        return sql
+
+    @classmethod
+    @telemetry.send_api_usage_telemetry(
+        project=_PROJECT,
+        func_params_to_log=[
+            "pip_requirements",
+            "external_access_integrations",
+            "target_instances",
+            "min_instances",
+            "enable_metrics",
+            "query_warehouse",
+            "runtime_environment",
+        ],
+    )
+    def register(
+        cls,
+        source: Union[str, Callable[_Args, _ReturnValue]],
+        compute_pool: str,
+        stage_name: str,
+        session: Optional[snowpark.Session] = None,
+        entrypoint: Optional[Union[str, list[str]]] = None,
+        target_instances: int = 1,
+        generate_suffix: bool = True,
+        **kwargs: Any,
+    ) -> "MLJobDefinition[_Args, _ReturnValue]":
+        # Use kwargs for less common optional parameters
+        database = kwargs.pop("database", None)
+        schema = kwargs.pop("schema", None)
+        min_instances = kwargs.pop("min_instances", target_instances)
+        pip_requirements = kwargs.pop("pip_requirements", None)
+        external_access_integrations = kwargs.pop("external_access_integrations", None)
+        env_vars = kwargs.pop("env_vars", None)
+        spec_overrides = kwargs.pop("spec_overrides", None)
+        enable_metrics = kwargs.pop("enable_metrics", True)
+        session = session or sp_context.get_active_session()
+        query_warehouse = kwargs.pop("query_warehouse", session.get_current_warehouse())
+        imports = kwargs.pop("imports", None)
+        runtime_environment = kwargs.pop(
+            "runtime_environment", os.environ.get(constants.RUNTIME_IMAGE_TAG_ENV_VAR, None)
+        )
+        overwrite = kwargs.pop("overwrite", False)
+        name = kwargs.pop("name", None)
+        # Warn if there are unknown kwargs
+        if kwargs:
+            logger.warning(f"Ignoring unknown kwargs: {kwargs.keys()}")
+
+        # Validate parameters
+        if database and not schema:
+            raise ValueError("Schema must be specified if database is specified.")
+        if target_instances < 1:
+            raise ValueError("target_instances must be greater than 0.")
+        if not (0 < min_instances <= target_instances):
+            raise ValueError("min_instances must be greater than 0 and less than or equal to target_instances.")
+        if min_instances > 1:
+            # Validate min_instances against compute pool max_nodes
+            pool_info = jb._get_compute_pool_info(session, compute_pool)
+            max_nodes = int(pool_info["max_nodes"])
+            if min_instances > max_nodes:
+                raise ValueError(
+                    f"The requested min_instances ({min_instances}) exceeds the max_nodes ({max_nodes}) "
+                    f"of compute pool '{compute_pool}'. Reduce min_instances or increase max_nodes."
+                )
+
+        if name:
+            parsed_database, parsed_schema, parsed_name = identifier.parse_schema_level_object_identifier(name)
+            database = parsed_database or database
+            schema = parsed_schema or schema
+            name = parsed_name
+        else:
+            name = payload_utils.get_payload_name(source, entrypoint)
+
+        # The logical identifier for this job definition (used in the stage path)
+        # is the resolved object name, not the fully qualified identifier.
+        job_definition_id = name if not generate_suffix else name + _generate_suffix()
+        stage_path_parts = identifier.parse_snowflake_stage_path(stage_name.lstrip("@"))
+        stage_name = f"@{'.'.join(filter(None, stage_path_parts[:3]))}"
+        stage_path = PurePosixPath(f"{stage_name}{stage_path_parts[-1].rstrip('/')}/{job_definition_id}")
+
+        try:
+            # Upload payload
+            uploaded_payload = payload_utils.JobPayload(
+                source, entrypoint=entrypoint, pip_requirements=pip_requirements, imports=imports
+            ).upload(session, stage_path, overwrite)
+        except SnowparkSQLException as e:
+            if e.sql_error_code == 90106:
+                raise RuntimeError(
+                    "Please specify a schema, either in the session context or as a parameter in the job submission"
+                )
+            raise
+
+        if runtime_environment is None and feature_flags.FeatureFlags.ENABLE_RUNTIME_VERSIONS.is_enabled(default=True):
+            # Pass a JSON object for runtime versions so it serializes as nested JSON in options
+            runtime_environment = json.dumps({"pythonVersion": f"{sys.version_info.major}.{sys.version_info.minor}"})
+
+        combined_env_vars = {**uploaded_payload.env_vars, **(env_vars or {})}
+        entrypoint_args = [v.as_posix() if isinstance(v, PurePath) else v for v in uploaded_payload.entrypoint]
+        spec_options = types.SpecOptions(
+            stage_path=stage_path.as_posix(),
+            # the args will be set at runtime
+            args=None,
+            env_vars=combined_env_vars,
+            enable_metrics=enable_metrics,
+            spec_overrides=spec_overrides,
+            runtime=runtime_environment if runtime_environment else None,
+            enable_stage_mount_v2=feature_flags.FeatureFlags.ENABLE_STAGE_MOUNT_V2.is_enabled(default=True),
+        )
+
+        job_options = types.JobOptions(
+            external_access_integrations=external_access_integrations,
+            query_warehouse=query_warehouse,
+            target_instances=target_instances,
+            min_instances=min_instances,
+            generate_suffix=generate_suffix,
+        )
+
+        return cls(
+            stage_name=stage_path.as_posix(),
+            spec_options=spec_options,
+            job_options=job_options,
+            compute_pool=compute_pool,
+            entrypoint_args=entrypoint_args,
+            session=session,
+            database=database,
+            schema=schema,
+            name=name,
+        )
+
+
+def _generate_suffix() -> str:
+    return str(uuid4().hex)[:8]