llama-stack 0.3.5__py3-none-any.whl → 0.4.0__py3-none-any.whl

llama_stack/providers/inline/batches/reference/batches.py

@@ -11,29 +11,44 @@ import json
 import time
 import uuid
 from io import BytesIO
-from typing import Any, Literal
+from typing import Any
 
 from openai.types.batch import BatchError, Errors
 from pydantic import BaseModel
 
-from llama_stack.apis.batches import Batches, BatchObject, ListBatchesResponse
-from llama_stack.apis.common.errors import ConflictError, ResourceNotFoundError
-from llama_stack.apis.files import Files, OpenAIFilePurpose
-from llama_stack.apis.inference import (
+from llama_stack.core.storage.kvstore import KVStore
+from llama_stack.log import get_logger
+from llama_stack_api import (
+    Batches,
+    BatchObject,
+    ConflictError,
+    Files,
     Inference,
+    ListBatchesResponse,
+    Models,
     OpenAIAssistantMessageParam,
     OpenAIChatCompletionRequestWithExtraBody,
     OpenAICompletionRequestWithExtraBody,
     OpenAIDeveloperMessageParam,
     OpenAIEmbeddingsRequestWithExtraBody,
+    OpenAIFilePurpose,
     OpenAIMessageParam,
     OpenAISystemMessageParam,
     OpenAIToolMessageParam,
     OpenAIUserMessageParam,
+    ResourceNotFoundError,
+)
+from llama_stack_api.batches.models import (
+    CancelBatchRequest,
+    CreateBatchRequest,
+    ListBatchesRequest,
+    RetrieveBatchRequest,
+)
+from llama_stack_api.files.models import (
+    RetrieveFileContentRequest,
+    RetrieveFileRequest,
+    UploadFileRequest,
 )
-from llama_stack.apis.models import Models
-from llama_stack.log import get_logger
-from llama_stack.providers.utils.kvstore import KVStore
 
 from .config import ReferenceBatchesImplConfig
 
@@ -136,11 +151,7 @@ class ReferenceBatchesImpl(Batches):
     # TODO (SECURITY): this currently works w/ configured api keys, not with x-llamastack-provider-data or with user policy restrictions
     async def create_batch(
         self,
-        input_file_id: str,
-        endpoint: str,
-        completion_window: Literal["24h"],
-        metadata: dict[str, str] | None = None,
-        idempotency_key: str | None = None,
+        request: CreateBatchRequest,
     ) -> BatchObject:
         """
         Create a new batch for processing multiple API requests.
@@ -181,14 +192,14 @@ class ReferenceBatchesImpl(Batches):
 
         # TODO: set expiration time for garbage collection
 
-        if endpoint not in ["/v1/chat/completions", "/v1/completions", "/v1/embeddings"]:
+        if request.endpoint not in ["/v1/chat/completions", "/v1/completions", "/v1/embeddings"]:
             raise ValueError(
-                f"Invalid endpoint: {endpoint}. Supported values: /v1/chat/completions, /v1/completions, /v1/embeddings. Code: invalid_value. Param: endpoint",
+                f"Invalid endpoint: {request.endpoint}. Supported values: /v1/chat/completions, /v1/completions, /v1/embeddings. Code: invalid_value. Param: endpoint",
             )
 
-        if completion_window != "24h":
+        if request.completion_window != "24h":
             raise ValueError(
-                f"Invalid completion_window: {completion_window}. Supported values are: 24h. Code: invalid_value. Param: completion_window",
+                f"Invalid completion_window: {request.completion_window}. Supported values are: 24h. Code: invalid_value. Param: completion_window",
             )
 
         batch_id = f"batch_{uuid.uuid4().hex[:16]}"
@@ -196,22 +207,22 @@ class ReferenceBatchesImpl(Batches):
         # For idempotent requests, use the idempotency key for the batch ID
         # This ensures the same key always maps to the same batch ID,
         # allowing us to detect parameter conflicts
-        if idempotency_key is not None:
-            hash_input = idempotency_key.encode("utf-8")
+        if request.idempotency_key is not None:
+            hash_input = request.idempotency_key.encode("utf-8")
             hash_digest = hashlib.sha256(hash_input).hexdigest()[:24]
             batch_id = f"batch_{hash_digest}"
 
             try:
-                existing_batch = await self.retrieve_batch(batch_id)
+                existing_batch = await self.retrieve_batch(RetrieveBatchRequest(batch_id=batch_id))
 
                 if (
-                    existing_batch.input_file_id != input_file_id
-                    or existing_batch.endpoint != endpoint
-                    or existing_batch.completion_window != completion_window
-                    or existing_batch.metadata != metadata
+                    existing_batch.input_file_id != request.input_file_id
+                    or existing_batch.endpoint != request.endpoint
+                    or existing_batch.completion_window != request.completion_window
+                    or existing_batch.metadata != request.metadata
                 ):
                     raise ConflictError(
-                        f"Idempotency key '{idempotency_key}' was previously used with different parameters. "
+                        f"Idempotency key '{request.idempotency_key}' was previously used with different parameters. "
                         "Either use a new idempotency key or ensure all parameters match the original request."
                     )
 
@@ -226,12 +237,12 @@ class ReferenceBatchesImpl(Batches):
         batch = BatchObject(
             id=batch_id,
             object="batch",
-            endpoint=endpoint,
-            input_file_id=input_file_id,
-            completion_window=completion_window,
+            endpoint=request.endpoint,
+            input_file_id=request.input_file_id,
+            completion_window=request.completion_window,
             status="validating",
             created_at=current_time,
-            metadata=metadata,
+            metadata=request.metadata,
         )
 
         await self.kvstore.set(f"batch:{batch_id}", batch.to_json())
@@ -243,28 +254,27 @@ class ReferenceBatchesImpl(Batches):
 
         return batch
 
-    async def cancel_batch(self, batch_id: str) -> BatchObject:
+    async def cancel_batch(self, request: CancelBatchRequest) -> BatchObject:
         """Cancel a batch that is in progress."""
-        batch = await self.retrieve_batch(batch_id)
+        batch = await self.retrieve_batch(RetrieveBatchRequest(batch_id=request.batch_id))
 
         if batch.status in ["cancelled", "cancelling"]:
             return batch
 
         if batch.status in ["completed", "failed", "expired"]:
-            raise ConflictError(f"Cannot cancel batch '{batch_id}' with status '{batch.status}'")
+            raise ConflictError(f"Cannot cancel batch '{request.batch_id}' with status '{batch.status}'")
 
-        await self._update_batch(batch_id, status="cancelling", cancelling_at=int(time.time()))
+        await self._update_batch(request.batch_id, status="cancelling", cancelling_at=int(time.time()))
 
-        if batch_id in self._processing_tasks:
-            self._processing_tasks[batch_id].cancel()
+        if request.batch_id in self._processing_tasks:
+            self._processing_tasks[request.batch_id].cancel()
             # note: task removal and status="cancelled" handled in finally block of _process_batch
 
-        return await self.retrieve_batch(batch_id)
+        return await self.retrieve_batch(RetrieveBatchRequest(batch_id=request.batch_id))
 
     async def list_batches(
         self,
-        after: str | None = None,
-        limit: int = 20,
+        request: ListBatchesRequest,
     ) -> ListBatchesResponse:
         """
         List all batches, eventually only for the current user.
@@ -281,14 +291,14 @@ class ReferenceBatchesImpl(Batches):
         batches.sort(key=lambda b: b.created_at, reverse=True)
 
         start_idx = 0
-        if after:
+        if request.after:
             for i, batch in enumerate(batches):
-                if batch.id == after:
+                if batch.id == request.after:
                     start_idx = i + 1
                     break
 
-        page_batches = batches[start_idx : start_idx + limit]
-        has_more = (start_idx + limit) < len(batches)
+        page_batches = batches[start_idx : start_idx + request.limit]
+        has_more = (start_idx + request.limit) < len(batches)
 
         first_id = page_batches[0].id if page_batches else None
         last_id = page_batches[-1].id if page_batches else None
@@ -300,11 +310,11 @@ class ReferenceBatchesImpl(Batches):
             has_more=has_more,
         )
 
-    async def retrieve_batch(self, batch_id: str) -> BatchObject:
+    async def retrieve_batch(self, request: RetrieveBatchRequest) -> BatchObject:
         """Retrieve information about a specific batch."""
-        batch_data = await self.kvstore.get(f"batch:{batch_id}")
+        batch_data = await self.kvstore.get(f"batch:{request.batch_id}")
         if not batch_data:
-            raise ResourceNotFoundError(batch_id, "Batch", "batches.list()")
+            raise ResourceNotFoundError(request.batch_id, "Batch", "batches.list()")
 
         return BatchObject.model_validate_json(batch_data)
 
@@ -312,7 +322,7 @@ class ReferenceBatchesImpl(Batches):
         """Update batch fields in kvstore."""
         async with self._update_batch_lock:
             try:
-                batch = await self.retrieve_batch(batch_id)
+                batch = await self.retrieve_batch(RetrieveBatchRequest(batch_id=batch_id))
 
                 # batch processing is async. once cancelling, only allow "cancelled" status updates
                 if batch.status == "cancelling" and updates.get("status") != "cancelled":
@@ -344,7 +354,7 @@ class ReferenceBatchesImpl(Batches):
         requests: list[BatchRequest] = []
         errors: list[BatchError] = []
         try:
-            await self.files_api.openai_retrieve_file(batch.input_file_id)
+            await self.files_api.openai_retrieve_file(RetrieveFileRequest(file_id=batch.input_file_id))
         except Exception:
             errors.append(
                 BatchError(
@@ -357,8 +367,13 @@ class ReferenceBatchesImpl(Batches):
             return errors, requests
 
         # TODO(SECURITY): do something about large files
-        file_content_response = await self.files_api.openai_retrieve_file_content(batch.input_file_id)
-        file_content = file_content_response.body.decode("utf-8")
+        file_content_response = await self.files_api.openai_retrieve_file_content(
+            RetrieveFileContentRequest(file_id=batch.input_file_id)
+        )
+        # Handle both bytes and memoryview types - convert to bytes unconditionally
+        # (bytes(x) returns x if already bytes, creates new bytes from memoryview otherwise)
+        body_bytes = bytes(file_content_response.body)
+        file_content = body_bytes.decode("utf-8")
         for line_num, line in enumerate(file_content.strip().split("\n"), 1):
             if line.strip():  # skip empty lines
                 try:
@@ -415,8 +430,8 @@ class ReferenceBatchesImpl(Batches):
                         )
                         valid = False
 
-                    if (body := request.get("body")) and isinstance(body, dict):
-                        if body.get("stream", False):
+                    if (request_body := request.get("body")) and isinstance(request_body, dict):
+                        if request_body.get("stream", False):
                             errors.append(
                                 BatchError(
                                     code="streaming_unsupported",
@@ -447,7 +462,7 @@ class ReferenceBatchesImpl(Batches):
                             ]
 
                         for param, expected_type, type_string in required_params:
-                            if param not in body:
+                            if param not in request_body:
                                 errors.append(
                                     BatchError(
                                         code="invalid_request",
@@ -457,7 +472,7 @@ class ReferenceBatchesImpl(Batches):
                                     )
                                 )
                                 valid = False
-                            elif not isinstance(body[param], expected_type):
+                            elif not isinstance(request_body[param], expected_type):
                                 errors.append(
                                     BatchError(
                                         code="invalid_request",
@@ -468,15 +483,15 @@ class ReferenceBatchesImpl(Batches):
                                 )
                                 valid = False
 
-                        if "model" in body and isinstance(body["model"], str):
+                        if "model" in request_body and isinstance(request_body["model"], str):
                             try:
-                                await self.models_api.get_model(body["model"])
+                                await self.models_api.get_model(request_body["model"])
                             except Exception:
                                 errors.append(
                                     BatchError(
                                         code="model_not_found",
                                         line=line_num,
-                                        message=f"Model '{body['model']}' does not exist or is not supported",
+                                        message=f"Model '{request_body['model']}' does not exist or is not supported",
                                         param="body.model",
                                     )
                                 )
@@ -484,14 +499,14 @@ class ReferenceBatchesImpl(Batches):
 
                     if valid:
                         assert isinstance(url, str), "URL must be a string"  # for mypy
-                        assert isinstance(body, dict), "Body must be a dictionary"  # for mypy
+                        assert isinstance(request_body, dict), "Body must be a dictionary"  # for mypy
                         requests.append(
                             BatchRequest(
                                 line_num=line_num,
                                 url=url,
                                 method=request["method"],
                                 custom_id=request["custom_id"],
-                                body=body,
+                                body=request_body,
                             ),
                         )
                 except json.JSONDecodeError:
@@ -529,7 +544,7 @@ class ReferenceBatchesImpl(Batches):
     async def _process_batch_impl(self, batch_id: str) -> None:
         """Implementation of batch processing logic."""
         errors: list[BatchError] = []
-        batch = await self.retrieve_batch(batch_id)
+        batch = await self.retrieve_batch(RetrieveBatchRequest(batch_id=batch_id))
 
         errors, requests = await self._validate_input(batch)
         if errors:
@@ -675,5 +690,8 @@ class ReferenceBatchesImpl(Batches):
 
         with AsyncBytesIO("\n".join(output_lines).encode("utf-8")) as file_buffer:
             file_buffer.filename = f"{batch_id}_{file_type}.jsonl"
-            uploaded_file = await self.files_api.openai_upload_file(file=file_buffer, purpose=OpenAIFilePurpose.BATCH)
+            uploaded_file = await self.files_api.openai_upload_file(
+                request=UploadFileRequest(purpose=OpenAIFilePurpose.BATCH),
+                file=file_buffer,
+            )
             return uploaded_file.id

llama_stack/providers/inline/datasetio/localfs/datasetio.py

@@ -5,13 +5,10 @@
 # the root directory of this source tree.
 from typing import Any
 
-from llama_stack.apis.common.responses import PaginatedResponse
-from llama_stack.apis.datasetio import DatasetIO
-from llama_stack.apis.datasets import Dataset
-from llama_stack.providers.datatypes import DatasetsProtocolPrivate
+from llama_stack.core.storage.kvstore import kvstore_impl
 from llama_stack.providers.utils.datasetio.url_utils import get_dataframe_from_uri
-from llama_stack.providers.utils.kvstore import kvstore_impl
 from llama_stack.providers.utils.pagination import paginate_records
+from llama_stack_api import Dataset, DatasetIO, DatasetsProtocolPrivate, PaginatedResponse
 
 from .config import LocalFSDatasetIOConfig
 

llama_stack/providers/inline/eval/meta_reference/eval.py

@@ -8,28 +8,27 @@ from typing import Any
 
 from tqdm import tqdm
 
-from llama_stack.apis.agents import Agents, StepType
-from llama_stack.apis.benchmarks import Benchmark
-from llama_stack.apis.datasetio import DatasetIO
-from llama_stack.apis.datasets import Datasets
-from llama_stack.apis.inference import (
+from llama_stack.core.storage.kvstore import kvstore_impl
+from llama_stack.providers.utils.common.data_schema_validator import ColumnName
+from llama_stack_api import (
+    Agents,
+    Benchmark,
+    BenchmarkConfig,
+    BenchmarksProtocolPrivate,
+    DatasetIO,
+    Datasets,
+    Eval,
+    EvaluateResponse,
     Inference,
+    Job,
+    JobStatus,
     OpenAIChatCompletionRequestWithExtraBody,
     OpenAICompletionRequestWithExtraBody,
     OpenAISystemMessageParam,
     OpenAIUserMessageParam,
-    UserMessage,
-)
-from llama_stack.apis.scoring import Scoring
-from llama_stack.providers.datatypes import BenchmarksProtocolPrivate
-from llama_stack.providers.inline.agents.meta_reference.agent_instance import (
-    MEMORY_QUERY_TOOL,
+    Scoring,
 )
-from llama_stack.providers.utils.common.data_schema_validator import ColumnName
-from llama_stack.providers.utils.kvstore import kvstore_impl
 
-from .....apis.common.job_types import Job, JobStatus
-from .....apis.eval.eval import BenchmarkConfig, Eval, EvaluateResponse
 from .config import MetaReferenceEvalConfig
 
 EVAL_TASKS_PREFIX = "benchmarks:"
@@ -118,49 +117,6 @@ class MetaReferenceEvalImpl(
         self.jobs[job_id] = res
         return Job(job_id=job_id, status=JobStatus.completed)
 
-    async def _run_agent_generation(
-        self, input_rows: list[dict[str, Any]], benchmark_config: BenchmarkConfig
-    ) -> list[dict[str, Any]]:
-        candidate = benchmark_config.eval_candidate
-        create_response = await self.agents_api.create_agent(candidate.config)
-        agent_id = create_response.agent_id
-
-        generations = []
-        for i, x in tqdm(enumerate(input_rows)):
-            assert ColumnName.chat_completion_input.value in x, "Invalid input row"
-            input_messages = json.loads(x[ColumnName.chat_completion_input.value])
-            input_messages = [UserMessage(**x) for x in input_messages if x["role"] == "user"]
-
-            # NOTE: only single-turn agent generation is supported. Create a new session for each input row
-            session_create_response = await self.agents_api.create_agent_session(agent_id, f"session-{i}")
-            session_id = session_create_response.session_id
-
-            turn_request = dict(
-                agent_id=agent_id,
-                session_id=session_id,
-                messages=input_messages,
-                stream=True,
-            )
-            turn_response = [chunk async for chunk in await self.agents_api.create_agent_turn(**turn_request)]
-            final_event = turn_response[-1].event.payload
-
-            # check if there's a memory retrieval step and extract the context
-            memory_rag_context = None
-            for step in final_event.turn.steps:
-                if step.step_type == StepType.tool_execution.value:
-                    for tool_response in step.tool_responses:
-                        if tool_response.tool_name == MEMORY_QUERY_TOOL:
-                            memory_rag_context = " ".join(x.text for x in tool_response.content)
-
-            agent_generation = {}
-            agent_generation[ColumnName.generated_answer.value] = final_event.turn.output_message.content
-            if memory_rag_context:
-                agent_generation[ColumnName.context.value] = memory_rag_context
-
-            generations.append(agent_generation)
-
-        return generations
-
     async def _run_model_generation(
         self, input_rows: list[dict[str, Any]], benchmark_config: BenchmarkConfig
     ) -> list[dict[str, Any]]:
@@ -215,9 +171,8 @@ class MetaReferenceEvalImpl(
         benchmark_config: BenchmarkConfig,
     ) -> EvaluateResponse:
         candidate = benchmark_config.eval_candidate
-        if candidate.type == "agent":
-            generations = await self._run_agent_generation(input_rows, benchmark_config)
-        elif candidate.type == "model":
+        # Agent evaluation removed
+        if candidate.type == "model":
             generations = await self._run_model_generation(input_rows, benchmark_config)
         else:
             raise ValueError(f"Invalid candidate type: {candidate.type}")

llama_stack/providers/inline/files/localfs/files.py

@@ -7,27 +7,30 @@
 import time
 import uuid
 from pathlib import Path
-from typing import Annotated
 
-from fastapi import Depends, File, Form, Response, UploadFile
+from fastapi import Response, UploadFile
 
-from llama_stack.apis.common.errors import ResourceNotFoundError
-from llama_stack.apis.common.responses import Order
-from llama_stack.apis.files import (
-    ExpiresAfter,
+from llama_stack.core.access_control.datatypes import Action
+from llama_stack.core.datatypes import AccessRule
+from llama_stack.core.id_generation import generate_object_id
+from llama_stack.core.storage.sqlstore.authorized_sqlstore import AuthorizedSqlStore
+from llama_stack.core.storage.sqlstore.sqlstore import sqlstore_impl
+from llama_stack.log import get_logger
+from llama_stack_api import (
+    DeleteFileRequest,
     Files,
+    ListFilesRequest,
     ListOpenAIFileResponse,
     OpenAIFileDeleteResponse,
     OpenAIFileObject,
     OpenAIFilePurpose,
+    Order,
+    ResourceNotFoundError,
+    RetrieveFileContentRequest,
+    RetrieveFileRequest,
+    UploadFileRequest,
 )
-from llama_stack.core.datatypes import AccessRule
-from llama_stack.core.id_generation import generate_object_id
-from llama_stack.log import get_logger
-from llama_stack.providers.utils.files.form_data import parse_expires_after
-from llama_stack.providers.utils.sqlstore.api import ColumnDefinition, ColumnType
-from llama_stack.providers.utils.sqlstore.authorized_sqlstore import AuthorizedSqlStore
-from llama_stack.providers.utils.sqlstore.sqlstore import sqlstore_impl
+from llama_stack_api.internal.sqlstore import ColumnDefinition, ColumnType
 
 from .config import LocalfsFilesImplConfig
 
@@ -72,12 +75,12 @@ class LocalfsFilesImpl(Files):
         """Get the filesystem path for a file ID."""
         return Path(self.config.storage_dir) / file_id
 
-    async def _lookup_file_id(self, file_id: str) -> tuple[OpenAIFileObject, Path]:
+    async def _lookup_file_id(self, file_id: str, action: Action = Action.READ) -> tuple[OpenAIFileObject, Path]:
         """Look up a OpenAIFileObject and filesystem path from its ID."""
         if not self.sql_store:
             raise RuntimeError("Files provider not initialized")
 
-        row = await self.sql_store.fetch_one("openai_files", where={"id": file_id})
+        row = await self.sql_store.fetch_one("openai_files", where={"id": file_id}, action=action)
         if not row:
             raise ResourceNotFoundError(file_id, "File", "client.files.list()")
 
@@ -87,14 +90,16 @@ class LocalfsFilesImpl(Files):
     # OpenAI Files API Implementation
     async def openai_upload_file(
         self,
-        file: Annotated[UploadFile, File()],
-        purpose: Annotated[OpenAIFilePurpose, Form()],
-        expires_after: Annotated[ExpiresAfter | None, Depends(parse_expires_after)] = None,
+        request: UploadFileRequest,
+        file: UploadFile,
     ) -> OpenAIFileObject:
         """Upload a file that can be used across various endpoints."""
         if not self.sql_store:
             raise RuntimeError("Files provider not initialized")
 
+        purpose = request.purpose
+        expires_after = request.expires_after
+
         if expires_after is not None:
             logger.warning(
                 f"File expiration is not supported by this provider, ignoring expires_after: {expires_after}"
@@ -136,15 +141,17 @@ class LocalfsFilesImpl(Files):
 
     async def openai_list_files(
         self,
-        after: str | None = None,
-        limit: int | None = 10000,
-        order: Order | None = Order.desc,
-        purpose: OpenAIFilePurpose | None = None,
+        request: ListFilesRequest,
     ) -> ListOpenAIFileResponse:
         """Returns a list of files that belong to the user's organization."""
         if not self.sql_store:
             raise RuntimeError("Files provider not initialized")
 
+        after = request.after
+        limit = request.limit
+        order = request.order
+        purpose = request.purpose
+
         if not order:
             order = Order.desc
 
@@ -179,16 +186,17 @@ class LocalfsFilesImpl(Files):
             last_id=files[-1].id if files else "",
         )
 
-    async def openai_retrieve_file(self, file_id: str) -> OpenAIFileObject:
+    async def openai_retrieve_file(self, request: RetrieveFileRequest) -> OpenAIFileObject:
         """Returns information about a specific file."""
-        file_obj, _ = await self._lookup_file_id(file_id)
+        file_obj, _ = await self._lookup_file_id(request.file_id)
 
         return file_obj
 
-    async def openai_delete_file(self, file_id: str) -> OpenAIFileDeleteResponse:
+    async def openai_delete_file(self, request: DeleteFileRequest) -> OpenAIFileDeleteResponse:
         """Delete a file."""
+        file_id = request.file_id
         # Delete physical file
-        _, file_path = await self._lookup_file_id(file_id)
+        _, file_path = await self._lookup_file_id(file_id, action=Action.DELETE)
         if file_path.exists():
             file_path.unlink()
 
@@ -201,14 +209,15 @@ class LocalfsFilesImpl(Files):
             deleted=True,
         )
 
-    async def openai_retrieve_file_content(self, file_id: str) -> Response:
+    async def openai_retrieve_file_content(self, request: RetrieveFileContentRequest) -> Response:
         """Returns the contents of the specified file."""
+        file_id = request.file_id
         # Read file content
         file_obj, file_path = await self._lookup_file_id(file_id)
 
         if not file_path.exists():
             logger.warning(f"File '{file_id}'s underlying '{file_path}' is missing, deleting metadata.")
-            await self.openai_delete_file(file_id)
+            await self.openai_delete_file(DeleteFileRequest(file_id=file_id))
             raise ResourceNotFoundError(file_id, "File", "client.files.list()")
 
         # Return as binary response with appropriate content type

llama_stack/providers/inline/inference/meta_reference/config.py

@@ -8,15 +8,15 @@ from typing import Any
 
 from pydantic import BaseModel, field_validator
 
-from llama_stack.apis.inference import QuantizationConfig
 from llama_stack.providers.utils.inference import supported_inference_models
+from llama_stack_api import QuantizationConfig
 
 
 class MetaReferenceInferenceConfig(BaseModel):
     # this is a placeholder to indicate inference model id
     # the actual inference model id is dtermined by the moddel id in the request
     # Note: you need to register the model before using it for inference
-    # models in the resouce list in the run.yaml config will be registered automatically
+    # models in the resouce list in the config.yaml config will be registered automatically
     model: str | None = None
     torch_seed: int | None = None
     max_seq_len: int = 4096