snowflake-ml-python 1.16.0__py3-none-any.whl → 1.18.0__py3-none-any.whl

snowflake/ml/jobs/job.py

@@ -12,12 +12,19 @@ from snowflake import snowpark
 from snowflake.ml._internal import telemetry
 from snowflake.ml._internal.utils import identifier
 from snowflake.ml._internal.utils.mixins import SerializableSessionMixin
-from snowflake.ml.jobs._utils import constants, interop_utils, query_helper, types
+from snowflake.ml.jobs._interop import results as interop_result, utils as interop_utils
+from snowflake.ml.jobs._utils import (
+    constants,
+    payload_utils,
+    query_helper,
+    stage_utils,
+    types,
+)
 from snowflake.snowpark import Row, context as sp_context
 from snowflake.snowpark.exceptions import SnowparkSQLException
 
 _PROJECT = "MLJob"
-TERMINAL_JOB_STATUSES = {"FAILED", "DONE", "CANCELLED", "INTERNAL_ERROR"}
+TERMINAL_JOB_STATUSES = {"FAILED", "DONE", "CANCELLED", "INTERNAL_ERROR", "DELETED"}
 
 T = TypeVar("T")
 
@@ -36,7 +43,12 @@ class MLJob(Generic[T], SerializableSessionMixin):
         self._session = session or sp_context.get_active_session()
 
         self._status: types.JOB_STATUS = "PENDING"
-        self._result: Optional[interop_utils.ExecutionResult] = None
+        self._result: Optional[interop_result.ExecutionResult] = None
+
+    @cached_property
+    def _service_info(self) -> types.ServiceInfo:
+        """Get the job's service info."""
+        return _resolve_service_info(self.id, self._session)
 
     @cached_property
     def name(self) -> str:
@@ -44,7 +56,7 @@ class MLJob(Generic[T], SerializableSessionMixin):
 
     @cached_property
     def target_instances(self) -> int:
-        return _get_target_instances(self._session, self.id)
+        return self._service_info.target_instances
 
     @cached_property
     def min_instances(self) -> int:
@@ -69,8 +81,7 @@ class MLJob(Generic[T], SerializableSessionMixin):
     @cached_property
     def _compute_pool(self) -> str:
         """Get the job's compute pool name."""
-        row = _get_service_info(self._session, self.id)
-        return cast(str, row["compute_pool"])
+        return self._service_info.compute_pool
 
     @property
     def _service_spec(self) -> dict[str, Any]:
@@ -82,7 +93,13 @@ class MLJob(Generic[T], SerializableSessionMixin):
     @property
     def _container_spec(self) -> dict[str, Any]:
         """Get the job's main container spec."""
-        containers = self._service_spec["spec"]["containers"]
+        try:
+            containers = self._service_spec["spec"]["containers"]
+        except SnowparkSQLException as e:
+            if e.sql_error_code == 2003:
+                # If the job is deleted, the service spec is not available
+                return {}
+            raise
         if len(containers) == 1:
             return cast(dict[str, Any], containers[0])
         try:
@@ -103,24 +120,30 @@ class MLJob(Generic[T], SerializableSessionMixin):
         """Get the job's result file location."""
         result_path_str = self._container_spec["env"].get(constants.RESULT_PATH_ENV_VAR)
         if result_path_str is None:
-            raise RuntimeError(f"Job {self.name} doesn't have a result path configured")
-
-        # If result path is relative, it is relative to the stage mount path
-        result_path = Path(result_path_str)
-        if not result_path.is_absolute():
-            return f"{self._stage_path}/{result_path.as_posix()}"
-
-        # If result path is absolute, it is relative to the stage mount path
+            raise NotImplementedError(f"Job {self.name} doesn't have a result path configured")
+
+        return self._transform_path(result_path_str)
+
+    def _transform_path(self, path_str: str) -> str:
+        """Transform a local path within the container to a stage path."""
+        path = payload_utils.resolve_path(path_str)
+        if isinstance(path, stage_utils.StagePath):
+            # Stage paths need no transformation
+            return path.as_posix()
+        if not path.is_absolute():
+            # Assume relative paths are relative to stage mount path
+            return f"{self._stage_path}/{path.as_posix()}"
+
+        # If result path is absolute, rebase it onto the stage mount path
+        # TODO: Rather than matching by name, use the longest mount path which matches
         volume_mounts = self._container_spec["volumeMounts"]
         stage_mount_str = next(v for v in volume_mounts if v.get("name") == constants.STAGE_VOLUME_NAME)["mountPath"]
         stage_mount = Path(stage_mount_str)
         try:
-            relative_path = result_path.relative_to(stage_mount)
+            relative_path = path.relative_to(stage_mount)
             return f"{self._stage_path}/{relative_path.as_posix()}"
         except ValueError:
-            raise ValueError(
-                f"Result path {result_path} is absolute, but should be relative to stage mount {stage_mount}"
-            )
+            raise ValueError(f"Result path {path} is absolute, but should be relative to stage mount {stage_mount}")
 
     @overload
     def get_logs(
@@ -165,7 +188,14 @@ class MLJob(Generic[T], SerializableSessionMixin):
         Returns:
             The job's execution logs.
         """
-        logs = _get_logs(self._session, self.id, limit, instance_id, self._container_spec["name"], verbose)
+        logs = _get_logs(
+            self._session,
+            self.id,
+            limit,
+            instance_id,
+            self._container_spec["name"] if "name" in self._container_spec else constants.DEFAULT_CONTAINER_NAME,
+            verbose,
+        )
         assert isinstance(logs, str)  # mypy
         if as_list:
             return logs.splitlines()
@@ -199,8 +229,22 @@ class MLJob(Generic[T], SerializableSessionMixin):
         Raises:
             TimeoutError: If the job does not complete within the specified timeout.
         """
-        delay = constants.JOB_POLL_INITIAL_DELAY_SECONDS  # Start with 100ms delay
         start_time = time.monotonic()
+        try:
+            # spcs_wait_for() is a synchronous query, it’s more effective to do polling with exponential
+            # backoff. If the job is running for a long time. We want a hybrid option: use spcs_wait_for()
+            # for the first 30 seconds, then switch to polling for long running jobs
+            min_timeout = (
+                int(min(timeout, constants.JOB_SPCS_TIMEOUT_SECONDS))
+                if timeout >= 0
+                else constants.JOB_SPCS_TIMEOUT_SECONDS
+            )
+            query_helper.run_query(self._session, f"call {self.id}!spcs_wait_for('DONE', {min_timeout})")
+            return self.status
+        except SnowparkSQLException:
+            # if the function does not support for this environment
+            pass
+        delay: float = float(constants.JOB_POLL_INITIAL_DELAY_SECONDS)  # Start with 5s delay
         warning_shown = False
         while (status := self.status) not in TERMINAL_JOB_STATUSES:
             elapsed = time.monotonic() - start_time
@@ -218,7 +262,6 @@ class MLJob(Generic[T], SerializableSessionMixin):
             delay = min(delay * 1.2, constants.JOB_POLL_MAX_DELAY_SECONDS)  # Exponential backoff
         return self.status
 
-    @snowpark._internal.utils.private_preview(version="1.8.2")
     @telemetry.send_api_usage_telemetry(project=_PROJECT, func_params_to_log=["timeout"])
     def result(self, timeout: float = -1) -> T:
         """
@@ -237,13 +280,13 @@ class MLJob(Generic[T], SerializableSessionMixin):
         if self._result is None:
             self.wait(timeout)
             try:
-                self._result = interop_utils.fetch_result(self._session, self._result_path)
+                self._result = interop_utils.load_result(
+                    self._result_path, session=self._session, path_transform=self._transform_path
+                )
             except Exception as e:
-                raise RuntimeError(f"Failed to retrieve result for job (id={self.name})") from e
+                raise RuntimeError(f"Failed to retrieve result for job, error: {e!r}") from e
 
-        if self._result.success:
-            return cast(T, self._result.result)
-        raise RuntimeError(f"Job execution failed (id={self.name})") from self._result.exception
+        return cast(T, self._result.get_value())
 
     @telemetry.send_api_usage_telemetry(project=_PROJECT)
     def cancel(self) -> None:
@@ -256,22 +299,28 @@ class MLJob(Generic[T], SerializableSessionMixin):
             self._session.sql(f"CALL {self.id}!spcs_cancel_job()").collect()
             logger.debug(f"Cancellation requested for job {self.id}")
         except SnowparkSQLException as e:
-            raise RuntimeError(f"Failed to cancel job {self.id}: {e.message}") from e
+            raise RuntimeError(f"Failed to cancel job, error: {e!r}") from e
 
 
 @telemetry.send_api_usage_telemetry(project=_PROJECT, func_params_to_log=["job_id", "instance_id"])
 def _get_status(session: snowpark.Session, job_id: str, instance_id: Optional[int] = None) -> types.JOB_STATUS:
     """Retrieve job or job instance execution status."""
-    if instance_id is not None:
-        # Get specific instance status
-        rows = session.sql("SHOW SERVICE INSTANCES IN SERVICE IDENTIFIER(?)", params=(job_id,)).collect()
-        for row in rows:
-            if row["instance_id"] == str(instance_id):
-                return cast(types.JOB_STATUS, row["status"])
-        raise ValueError(f"Instance {instance_id} not found in job {job_id}")
-    else:
-        row = _get_service_info(session, job_id)
-        return cast(types.JOB_STATUS, row["status"])
+    try:
+        if instance_id is not None:
+            # Get specific instance status
+            rows = query_helper.run_query(session, "SHOW SERVICE INSTANCES IN SERVICE IDENTIFIER(?)", params=(job_id,))
+            for row in rows:
+                if row["instance_id"] == str(instance_id):
+                    return cast(types.JOB_STATUS, row["status"])
+            raise ValueError(f"Instance {instance_id} not found in job {job_id}")
+        else:
+            row = _get_service_info(session, job_id)
+            return cast(types.JOB_STATUS, row["status"])
+    except SnowparkSQLException as e:
+        if e.sql_error_code == 2003:
+            row = _get_service_info_spcs(session, job_id)
+            return cast(types.JOB_STATUS, row["STATUS"])
+        raise
 
 
 @telemetry.send_api_usage_telemetry(project=_PROJECT, func_params_to_log=["job_id"])
@@ -542,8 +591,21 @@ def _get_compute_pool_info(session: snowpark.Session, compute_pool: str) -> Row:
 
 @telemetry.send_api_usage_telemetry(project=_PROJECT, func_params_to_log=["job_id"])
 def _get_target_instances(session: snowpark.Session, job_id: str) -> int:
-    row = _get_service_info(session, job_id)
-    return int(row["target_instances"])
+    try:
+        row = _get_service_info(session, job_id)
+        return int(row["target_instances"])
+    except SnowparkSQLException as e:
+        if e.sql_error_code == 2003:
+            row = _get_service_info_spcs(session, job_id)
+            try:
+                params = json.loads(row["PARAMETERS"])
+                if isinstance(params, dict):
+                    return int(params.get("REPLICAS", 1))
+                else:
+                    return 1
+            except (json.JSONDecodeError, ValueError):
+                return 1
+        raise
 
 
 def _get_logs_spcs(
@@ -581,3 +643,87 @@ def _get_logs_spcs(
         query.append(f" LIMIT {limit};")
     rows = session.sql("\n".join(query)).collect()
     return rows
+
+
+def _get_service_info_spcs(session: snowpark.Session, job_id: str) -> Any:
+    """
+    Retrieve the service info from the SPCS interface.
+
+    Args:
+        session (Session): The Snowpark session to use.
+        job_id (str): The job ID.
+
+    Returns:
+        Any: The service info.
+
+    Raises:
+        SnowparkSQLException: If the job does not exist or is too old to retrieve.
+    """
+    db, schema, name = identifier.parse_schema_level_object_identifier(job_id)
+    db = db or session.get_current_database()
+    schema = schema or session.get_current_schema()
+    rows = query_helper.run_query(
+        session,
+        """
+        select DATABASE_NAME, SCHEMA_NAME, NAME, STATUS, COMPUTE_POOL_NAME, PARAMETERS
+        from table(snowflake.spcs.get_job_history())
+        where database_name = ? and schema_name = ? and name = ?
+        """,
+        params=(db, schema, name),
+    )
+    if rows:
+        return rows[0]
+    else:
+        raise SnowparkSQLException(f"Job {job_id} does not exist or could not be retrieved", sql_error_code=2003)
+
+
+def _resolve_service_info(id: str, session: snowpark.Session) -> types.ServiceInfo:
+    try:
+        row = _get_service_info(session, id)
+    except SnowparkSQLException as e:
+        if e.sql_error_code == 2003:
+            row = _get_service_info_spcs(session, id)
+        else:
+            raise
+    if not row:
+        raise SnowparkSQLException(f"Job {id} does not exist or could not be retrieved", sql_error_code=2003)
+
+    if "compute_pool" in row:
+        compute_pool = row["compute_pool"]
+    elif "COMPUTE_POOL_NAME" in row:
+        compute_pool = row["COMPUTE_POOL_NAME"]
+    else:
+        raise ValueError(f"compute_pool not found in row: {row}")
+
+    if "status" in row:
+        status = row["status"]
+    elif "STATUS" in row:
+        status = row["STATUS"]
+    else:
+        raise ValueError(f"status not found in row: {row}")
+    # Normalize target_instances
+    target_instances: int
+    if "target_instances" in row and row["target_instances"] is not None:
+        try:
+            target_instances = int(row["target_instances"])
+        except (ValueError, TypeError):
+            target_instances = 1
+    elif "PARAMETERS" in row and row["PARAMETERS"]:
+        try:
+            params = json.loads(row["PARAMETERS"])
+            target_instances = int(params.get("REPLICAS", 1)) if isinstance(params, dict) else 1
+        except (json.JSONDecodeError, ValueError, TypeError):
+            target_instances = 1
+    else:
+        target_instances = 1
+
+    database_name = row["database_name"] if "database_name" in row else row["DATABASE_NAME"]
+    schema_name = row["schema_name"] if "schema_name" in row else row["SCHEMA_NAME"]
+
+    return types.ServiceInfo(
+        database_name=database_name,
+        schema_name=schema_name,
+        status=cast(types.JOB_STATUS, status),
+        compute_pool=cast(str, compute_pool),
+        target_instances=target_instances,
+    )

snowflake/ml/jobs/manager.py

@@ -21,6 +21,7 @@ from snowflake.ml.jobs._utils import (
     spec_utils,
     types,
 )
+from snowflake.snowpark._internal import utils as sp_utils
 from snowflake.snowpark.context import get_active_session
 from snowflake.snowpark.exceptions import SnowparkSQLException
 from snowflake.snowpark.functions import coalesce, col, lit, when
@@ -179,8 +180,10 @@ def get_job(job_id: str, session: Optional[snowpark.Session] = None) -> jb.MLJob
         _ = job._service_spec
         return job
     except SnowparkSQLException as e:
-        if "does not exist" in e.message:
-            raise ValueError(f"Job does not exist: {job_id}") from e
+        if e.sql_error_code == 2003:
+            job = jb.MLJob[Any](job_id, session=session)
+            _ = job.status
+            return job
         raise
 
 
@@ -446,7 +449,7 @@ def _submit_job(
     Raises:
         ValueError: If database or schema value(s) are invalid
         RuntimeError: If schema is not specified in session context or job submission
-        snowpark.exceptions.SnowparkSQLException: if failed to upload payload
+        SnowparkSQLException: if failed to upload payload
     """
     session = _ensure_session(session)
 
@@ -512,49 +515,44 @@ def _submit_job(
         uploaded_payload = payload_utils.JobPayload(
             source, entrypoint=entrypoint, pip_requirements=pip_requirements, additional_payloads=imports
         ).upload(session, stage_path)
-    except snowpark.exceptions.SnowparkSQLException as e:
+    except SnowparkSQLException as e:
         if e.sql_error_code == 90106:
             raise RuntimeError(
                 "Please specify a schema, either in the session context or as a parameter in the job submission"
             )
         raise
 
-    # FIXME: Temporary patches, remove this after v1 is deprecated
-    if target_instances > 1:
-        default_spec_overrides = {
-            "spec": {
-                "endpoints": [
-                    {"name": "ray-dashboard-endpoint", "port": 12003, "protocol": "TCP"},
-                ]
-            },
-        }
-        if spec_overrides:
-            spec_overrides = spec_utils.merge_patch(
-                default_spec_overrides, spec_overrides, display_name="spec_overrides"
-            )
-        else:
-            spec_overrides = default_spec_overrides
-
-    if feature_flags.FeatureFlags.USE_SUBMIT_JOB_V2.is_enabled():
+    if feature_flags.FeatureFlags.USE_SUBMIT_JOB_V2.is_enabled(default=True):
         # Add default env vars (extracted from spec_utils.generate_service_spec)
         combined_env_vars = {**uploaded_payload.env_vars, **(env_vars or {})}
 
-        return _do_submit_job_v2(
-            session=session,
-            payload=uploaded_payload,
-            args=args,
-            env_vars=combined_env_vars,
-            spec_overrides=spec_overrides,
-            compute_pool=compute_pool,
-            job_id=job_id,
-            external_access_integrations=external_access_integrations,
-            query_warehouse=query_warehouse,
-            target_instances=target_instances,
-            min_instances=min_instances,
-            enable_metrics=enable_metrics,
-            use_async=True,
-            runtime_environment=runtime_environment,
-        )
+        try:
+            return _do_submit_job_v2(
+                session=session,
+                payload=uploaded_payload,
+                args=args,
+                env_vars=combined_env_vars,
+                spec_overrides=spec_overrides,
+                compute_pool=compute_pool,
+                job_id=job_id,
+                external_access_integrations=external_access_integrations,
+                query_warehouse=query_warehouse,
+                target_instances=target_instances,
+                min_instances=min_instances,
+                enable_metrics=enable_metrics,
+                use_async=True,
+                runtime_environment=runtime_environment,
+            )
+        except SnowparkSQLException as e:
+            if not (e.sql_error_code == 90237 and sp_utils.is_in_stored_procedure()):  # type: ignore[no-untyped-call]
+                raise
+            # SNOW-2390287: SYSTEM$EXECUTE_ML_JOB() is erroneously blocked in owner's rights
+            # stored procedures. This will be fixed in an upcoming release.
+            logger.warning(
+                "Job submission using V2 failed with error {}. Falling back to V1.".format(
+                    str(e).split("\n", 1)[0],
+                )
+            )
 
     # Fall back to v1
     # Generate service spec
@@ -688,7 +686,7 @@ def _do_submit_job_v2(
     # for the image tag or full image URL, we use that directly
     if runtime_environment:
         spec_options["RUNTIME"] = runtime_environment
-    elif feature_flags.FeatureFlags.ENABLE_IMAGE_VERSION_ENV_VAR.is_enabled():
+    elif feature_flags.FeatureFlags.ENABLE_RUNTIME_VERSIONS.is_enabled():
         # when feature flag is enabled, we get the local python version and wrap it in a dict
         # in system function, we can know whether it is python version or image tag or full image URL through the format
         spec_options["RUNTIME"] = json.dumps({"pythonVersion": f"{sys.version_info.major}.{sys.version_info.minor}"})
@@ -699,10 +697,21 @@ def _do_submit_job_v2(
         "MIN_INSTANCES": min_instances,
         "ASYNC": use_async,
     }
+    if payload.payload_name:
+        job_options["GENERATE_SUFFIX"] = True
     job_options = {k: v for k, v in job_options.items() if v is not None}
 
     query_template = "CALL SYSTEM$EXECUTE_ML_JOB(?, ?, ?, ?)"
-    params = [job_id, compute_pool, json.dumps(spec_options), json.dumps(job_options)]
+    if job_id:
+        database, schema, _ = identifier.parse_schema_level_object_identifier(job_id)
+    params = [
+        job_id
+        if payload.payload_name is None
+        else identifier.get_schema_level_object_identifier(database, schema, payload.payload_name) + "_",
+        compute_pool,
+        json.dumps(spec_options),
+        json.dumps(job_options),
+    ]
     actual_job_id = query_helper.run_query(session, query_template, params=params)[0][0]
 
     return get_job(actual_job_id, session=session)

snowflake/ml/model/__init__.py

@@ -1,3 +1,6 @@
+import sys
+import warnings
+
 from snowflake.ml.model._client.model.batch_inference_specs import (
     JobSpec,
     OutputSpec,
@@ -18,3 +21,19 @@ __all__ = [
     "SaveMode",
     "Volatility",
 ]
+
+_deprecation_warning_msg_for_3_9 = (
+    "Python 3.9 is deprecated in snowflake-ml-python. " "Please upgrade to Python 3.10 or greater."
+)
+
+warnings.filterwarnings(
+    "once",
+    message=_deprecation_warning_msg_for_3_9,
+)
+
+if sys.version_info.major == 3 and sys.version_info.minor == 9:
+    warnings.warn(
+        _deprecation_warning_msg_for_3_9,
+        category=DeprecationWarning,
+        stacklevel=2,
+    )

snowflake/ml/model/_client/model/batch_inference_specs.py

@@ -19,11 +19,74 @@ class SaveMode(str, Enum):
 
 
 class OutputSpec(BaseModel):
+    """Specification for batch inference output.
+
+    Defines where the inference results should be written and how to handle
+    existing files at the output location.
+
+    Attributes:
+        stage_location (str): The stage path where batch inference results will be saved.
+            This should be a full path including the stage with @ prefix. For example,
+            '@My_DB.PUBLIC.MY_STAGE/someth/path/'. A non-existent directory will be re-created.
+            Only Snowflake internal stages are supported at this moment.
+        mode (SaveMode): The save mode that determines behavior when files already exist
+            at the output location. Defaults to SaveMode.ERROR which raises an error
+            if files exist. Can be set to SaveMode.OVERWRITE to replace existing files.
+
+    Example:
+        >>> output_spec = OutputSpec(
+        ...     stage_location="@My_DB.PUBLIC.MY_STAGE/someth/path/",
+        ...     mode=SaveMode.OVERWRITE
+        ... )
+    """
+
     stage_location: str
     mode: SaveMode = SaveMode.ERROR
 
 
 class JobSpec(BaseModel):
+    """Specification for batch inference job execution.
+
+    Defines the compute resources, job settings, and execution parameters
+    for running batch inference jobs in Snowflake.
+
+    Attributes:
+        image_repo (Optional[str]): Container image repository for the inference job.
+            If not specified, uses the default repository.
+        job_name (Optional[str]): Custom name for the batch inference job.
+            If not provided, a name will be auto-generated in the form of "BATCH_INFERENCE_<UUID>".
+        num_workers (Optional[int]): The number of workers to run the inference service for handling
+            requests in parallel within an instance of the service. By default, it is set to 2*vCPU+1
+            of the node for CPU based inference and 1 for GPU based inference. For GPU based inference,
+            please see best practices before playing with this value.
+        function_name (Optional[str]): Name of the specific function to call for inference.
+            Required when the model has multiple inference functions.
+        force_rebuild (bool): Whether to force rebuilding the container image even if
+            it already exists. Defaults to False.
+        max_batch_rows (int): Maximum number of rows to process in a single batch.
+            Defaults to 1024. Larger values may improve throughput.
+        warehouse (Optional[str]): Snowflake warehouse to use for the batch inference job.
+            If not specified, uses the session's current warehouse.
+        cpu_requests (Optional[str]): The cpu limit for CPU based inference. Can be an integer,
+            fractional or string values. If None, we attempt to utilize all the vCPU of the node.
+        memory_requests (Optional[str]): The memory limit for inference. Can be an integer
+            or a fractional value, but requires a unit (GiB, MiB). If None, we attempt to utilize all
+            the memory of the node.
+        gpu_requests (Optional[str]): The gpu limit for GPU based inference. Can be integer or
+            string values. Use CPU if None.
+        replicas (Optional[int]): Number of job replicas to run for high availability.
+            If not specified, defaults to 1 replica.
+
+    Example:
+        >>> job_spec = JobSpec(
+        ...     job_name="my_inference_job",
+        ...     num_workers=4,
+        ...     cpu_requests="2",
+        ...     memory_requests="8Gi",
+        ...     max_batch_rows=2048
+        ... )
+    """
+
     image_repo: Optional[str] = None
     job_name: Optional[str] = None
     num_workers: Optional[int] = None

snowflake/ml/model/_client/model/inference_engine_utils.py

@@ -6,13 +6,9 @@ from snowflake.ml.model._client.ops import service_ops
 def _get_inference_engine_args(
     experimental_options: Optional[dict[str, Any]],
 ) -> Optional[service_ops.InferenceEngineArgs]:
-
-    if not experimental_options:
+    if not experimental_options or "inference_engine" not in experimental_options:
         return None
 
-    if "inference_engine" not in experimental_options:
-        raise ValueError("inference_engine is required in experimental_options")
-
     return service_ops.InferenceEngineArgs(
         inference_engine=experimental_options["inference_engine"],
         inference_engine_args_override=experimental_options.get("inference_engine_args_override"),