pixeltable 0.3.14__py3-none-any.whl → 0.5.7__py3-none-any.whl

pixeltable/functions/openai.py

@@ -1,8 +1,8 @@
 """
-Pixeltable [UDFs](https://pixeltable.readme.io/docs/user-defined-functions-udfs)
+Pixeltable UDFs
 that wrap various endpoints from the OpenAI API. In order to use them, you must
 first `pip install openai` and configure your OpenAI credentials, as described in
-the [Working with OpenAI](https://pixeltable.readme.io/docs/working-with-openai) tutorial.
+the [Working with OpenAI](https://docs.pixeltable.com/notebooks/integrations/working-with-openai) tutorial.
 """
 
 import base64
@@ -13,18 +13,19 @@ import logging
 import math
 import pathlib
 import re
-import uuid
-from typing import TYPE_CHECKING, Any, Callable, Literal, Optional, Type, TypeVar, Union, cast
+from typing import TYPE_CHECKING, Any, Callable, Type
 
 import httpx
 import numpy as np
 import PIL
 
 import pixeltable as pxt
-import pixeltable.type_system as ts
-from pixeltable import env, exprs
+from pixeltable import env, exprs, type_system as ts
+from pixeltable.config import Config
 from pixeltable.func import Batch, Tools
 from pixeltable.utils.code import local_public_names
+from pixeltable.utils.local_store import TempStore
+from pixeltable.utils.system import set_file_descriptor_limit
 
 if TYPE_CHECKING:
     import openai
@@ -33,13 +34,28 @@ _logger = logging.getLogger('pixeltable')
 
 
 @env.register_client('openai')
-def _(api_key: str) -> 'openai.AsyncOpenAI':
+def _(api_key: str, base_url: str | None = None, api_version: str | None = None) -> 'openai.AsyncOpenAI':
     import openai
 
+    max_connections = Config.get().get_int_value('openai.max_connections') or 2000
+    max_keepalive_connections = Config.get().get_int_value('openai.max_keepalive_connections') or 100
+    set_file_descriptor_limit(max_connections * 2)
+    default_query = None if api_version is None else {'api-version': api_version}
+
+    # Pixeltable scheduler's retry logic takes into account the rate limit-related response headers, so in theory we can
+    # benefit from disabling retries in the OpenAI client (max_retries=0). However to do that, we need to get smarter
+    # about idempotency keys and possibly more.
     return openai.AsyncOpenAI(
         api_key=api_key,
+        base_url=base_url,
+        default_query=default_query,
         # recommended to increase limits for async client to avoid connection errors
-        http_client=httpx.AsyncClient(limits=httpx.Limits(max_keepalive_connections=100, max_connections=500)),
+        http_client=httpx.AsyncClient(
+            limits=httpx.Limits(max_keepalive_connections=max_keepalive_connections, max_connections=max_connections),
+            # HTTP1 tends to perform better on this kind of workloads
+            http2=False,
+            http1=True,
+        ),
     )
 
 
@@ -89,6 +105,99 @@ def _rate_limits_pool(model: str) -> str:
     return f'rate-limits:openai:{model}'
 
 
+def _parse_header_duration(duration_str: str) -> float | None:
+    """Parses the value of x-ratelimit-reset-* header into seconds.
+
+    Returns None if the input cannot be parsed.
+
+    Real life examples of header values:
+    * '1m33.792s'
+    * '857ms'
+    * '0s'
+    * '47.874s'
+    * '156h58m48.601s'
+    """
+    if duration_str is None or duration_str.strip() == '':
+        return None
+    units = {
+        86400: r'(\d+)d',  # days
+        3600: r'(\d+)h',  # hours
+        60: r'(\d+)m(?:[^s]|$)',  # minutes
+        1: r'([\d.]+)s',  # seconds
+        0.001: r'(\d+)ms',  # millis
+    }
+    seconds = None
+    for unit_value, pattern in units.items():
+        match = re.search(pattern, duration_str)
+        if match:
+            seconds = seconds or 0.0
+            seconds += float(match.group(1)) * unit_value
+    _logger.debug(f'Parsed duration header value "{duration_str}" into {seconds} seconds')
+    return seconds
+
+
+def _get_header_info(
+    headers: httpx.Headers,
+) -> tuple[tuple[int, int, datetime.datetime] | None, tuple[int, int, datetime.datetime] | None]:
+    """Parses rate limit related headers"""
+    # Requests and project-requests are two separate limits of requests per minute. project-requests headers will be
+    # present if an RPM limit is configured on the project limit.
+    requests_info = _get_resource_info(headers, 'requests')
+    requests_fraction_remaining = _fract_remaining(requests_info)
+    project_requests_info = _get_resource_info(headers, 'project-requests')
+    project_requests_fraction_remaining = _fract_remaining(project_requests_info)
+
+    # If both limit infos are present, pick the one with the least percentage remaining
+    best_requests_info = requests_info or project_requests_info
+    if (
+        requests_fraction_remaining is not None
+        and project_requests_fraction_remaining is not None
+        and project_requests_fraction_remaining < requests_fraction_remaining
+    ):
+        best_requests_info = project_requests_info
+
+    # Same story with tokens
+    tokens_info = _get_resource_info(headers, 'tokens')
+    tokens_fraction_remaining = _fract_remaining(tokens_info)
+    project_tokens_info = _get_resource_info(headers, 'project-tokens')
+    project_tokens_fraction_remaining = _fract_remaining(project_tokens_info)
+
+    best_tokens_info = tokens_info or project_tokens_info
+    if (
+        tokens_fraction_remaining is not None
+        and project_tokens_fraction_remaining is not None
+        and project_tokens_fraction_remaining < tokens_fraction_remaining
+    ):
+        best_tokens_info = project_tokens_info
+
+    if best_requests_info is None or best_tokens_info is None:
+        _logger.debug(f'get_header_info(): incomplete rate limit info: {headers}')
+
+    return best_requests_info, best_tokens_info
+
+
+def _get_resource_info(headers: httpx.Headers, resource: str) -> tuple[int, int, datetime.datetime] | None:
+    remaining_str = headers.get(f'x-ratelimit-remaining-{resource}')
+    if remaining_str is None:
+        return None
+    remaining = int(remaining_str)
+    limit_str = headers.get(f'x-ratelimit-limit-{resource}')
+    limit = int(limit_str) if limit_str is not None else None
+    reset_str = headers.get(f'x-ratelimit-reset-{resource}')
+    reset_in_seconds = _parse_header_duration(reset_str) or 5.0  # Default to 5 seconds
+    reset_ts = datetime.datetime.now(tz=datetime.timezone.utc) + datetime.timedelta(seconds=reset_in_seconds)
+    return (limit, remaining, reset_ts)
+
+
+def _fract_remaining(resource_info: tuple[int, int, datetime.datetime] | None) -> float | None:
+    if resource_info is None:
+        return None
+    limit, remaining, _ = resource_info
+    if limit is None or remaining is None:
+        return None
+    return remaining / limit
+
+
 class OpenAIRateLimitsInfo(env.RateLimitsInfo):
     retryable_errors: tuple[Type[Exception], ...]
 
@@ -109,61 +218,36 @@ class OpenAIRateLimitsInfo(env.RateLimitsInfo):
             openai.InternalServerError,
         )
 
-    def get_retry_delay(self, exc: Exception) -> Optional[float]:
+    def record_exc(self, request_ts: datetime.datetime, exc: Exception) -> None:
         import openai
 
-        if not isinstance(exc, self.retryable_errors):
-            return None
-        assert isinstance(exc, openai.APIError)
-        return 1.0
-
-
-# RE pattern for duration in '*-reset' headers;
-# examples: 1d2h3ms, 4m5.6s; # fractional seconds can be reported as 0.5s or 500ms
-_header_duration_pattern = re.compile(r'(?:(\d+)d)?(?:(\d+)h)?(?:(\d+)ms)|(?:(\d+)m)?(?:([\d.]+)s)?')
-
-
-def _parse_header_duration(duration_str: str) -> datetime.timedelta:
-    match = _header_duration_pattern.match(duration_str)
-    if not match:
-        raise ValueError('Invalid duration format')
+        _ = isinstance(exc, openai.APIError)
+        if not isinstance(exc, openai.APIError) or not hasattr(exc, 'response') or not hasattr(exc.response, 'headers'):
+            return
 
-    days = int(match.group(1) or 0)
-    hours = int(match.group(2) or 0)
-    milliseconds = int(match.group(3) or 0)
-    minutes = int(match.group(4) or 0)
-    seconds = float(match.group(5) or 0)
+        requests_info, tokens_info = _get_header_info(exc.response.headers)
+        _logger.debug(
+            f'record_exc(): request_ts: {request_ts}, requests_info={requests_info} tokens_info={tokens_info}'
+        )
+        self.record(request_ts=request_ts, requests=requests_info, tokens=tokens_info)
+        self.has_exc = True
 
-    return datetime.timedelta(days=days, hours=hours, minutes=minutes, seconds=seconds, milliseconds=milliseconds)
+    def _retry_delay_from_exception(self, exc: Exception) -> float | None:
+        try:
+            retry_after_str = exc.response.headers.get('retry-after')  # type: ignore
+        except AttributeError:
+            return None
+        if retry_after_str is not None and re.fullmatch(r'\d{1,4}', retry_after_str):
+            return float(retry_after_str)
+        return None
 
+    def get_retry_delay(self, exc: Exception, attempt: int) -> float | None:
+        import openai
 
-def _get_header_info(
-    headers: httpx.Headers, *, requests: bool = True, tokens: bool = True
-) -> tuple[Optional[tuple[int, int, datetime.datetime]], Optional[tuple[int, int, datetime.datetime]]]:
-    assert requests or tokens
-    now = datetime.datetime.now(tz=datetime.timezone.utc)
-
-    requests_info: Optional[tuple[int, int, datetime.datetime]] = None
-    if requests:
-        requests_limit_str = headers.get('x-ratelimit-limit-requests')
-        requests_limit = int(requests_limit_str) if requests_limit_str is not None else None
-        requests_remaining_str = headers.get('x-ratelimit-remaining-requests')
-        requests_remaining = int(requests_remaining_str) if requests_remaining_str is not None else None
-        requests_reset_str = headers.get('x-ratelimit-reset-requests')
-        requests_reset_ts = now + _parse_header_duration(requests_reset_str)
-        requests_info = (requests_limit, requests_remaining, requests_reset_ts)
-
-    tokens_info: Optional[tuple[int, int, datetime.datetime]] = None
-    if tokens:
-        tokens_limit_str = headers.get('x-ratelimit-limit-tokens')
-        tokens_limit = int(tokens_limit_str) if tokens_limit_str is not None else None
-        tokens_remaining_str = headers.get('x-ratelimit-remaining-tokens')
-        tokens_remaining = int(tokens_remaining_str) if tokens_remaining_str is not None else None
-        tokens_reset_str = headers.get('x-ratelimit-reset-tokens')
-        tokens_reset_ts = now + _parse_header_duration(tokens_reset_str)
-        tokens_info = (tokens_limit, tokens_remaining, tokens_reset_ts)
-
-    return requests_info, tokens_info
+        if not isinstance(exc, self.retryable_errors):
+            return None
+        assert isinstance(exc, openai.APIError)
+        return self._retry_delay_from_exception(exc) or super().get_retry_delay(exc, attempt)
 
 
 #####################################
@@ -171,15 +255,7 @@ def _get_header_info(
 
 
 @pxt.udf
-async def speech(
-    input: str,
-    *,
-    model: str,
-    voice: str,
-    response_format: Optional[str] = None,
-    speed: Optional[float] = None,
-    timeout: Optional[float] = None,
-) -> pxt.Audio:
+async def speech(input: str, *, model: str, voice: str, model_kwargs: dict[str, Any] | None = None) -> pxt.Audio:
     """
     Generates audio from the input text.
 
@@ -199,8 +275,8 @@ async def speech(
         model: The model to use for speech synthesis.
         voice: The voice profile to use for speech synthesis. Supported options include:
             `alloy`, `echo`, `fable`, `onyx`, `nova`, and `shimmer`.
-
-    For details on the other parameters, see: <https://platform.openai.com/docs/api-reference/audio/createSpeech>
+        model_kwargs: Additional keyword args for the OpenAI `audio/speech` API. For details on the available
+            parameters, see: <https://platform.openai.com/docs/api-reference/audio/createSpeech>
 
     Returns:
         An audio file containing the synthesized speech.
@@ -211,30 +287,18 @@ async def speech(
 
         >>> tbl.add_computed_column(audio=speech(tbl.text, model='tts-1', voice='nova'))
     """
-    content = await _openai_client().audio.speech.create(
-        input=input,
-        model=model,
-        voice=voice,  # type: ignore
-        response_format=_opt(response_format),  # type: ignore
-        speed=_opt(speed),
-        timeout=_opt(timeout),
-    )
-    ext = response_format or 'mp3'
-    output_filename = str(env.Env.get().tmp_dir / f'{uuid.uuid4()}.{ext}')
+    if model_kwargs is None:
+        model_kwargs = {}
+
+    content = await _openai_client().audio.speech.create(input=input, model=model, voice=voice, **model_kwargs)
+    ext = model_kwargs.get('response_format', 'mp3')
+    output_filename = str(TempStore.create_path(extension=f'.{ext}'))
     content.write_to_file(output_filename)
     return output_filename
 
 
 @pxt.udf
-async def transcriptions(
-    audio: pxt.Audio,
-    *,
-    model: str,
-    language: Optional[str] = None,
-    prompt: Optional[str] = None,
-    temperature: Optional[float] = None,
-    timeout: Optional[float] = None,
-) -> dict:
+async def transcriptions(audio: pxt.Audio, *, model: str, model_kwargs: dict[str, Any] | None = None) -> dict:
     """
     Transcribes audio into the input language.
 
@@ -252,8 +316,8 @@ async def transcriptions(
     Args:
         audio: The audio to transcribe.
         model: The model to use for speech transcription.
-
-    For details on the other parameters, see: <https://platform.openai.com/docs/api-reference/audio/createTranscription>
+        model_kwargs: Additional keyword args for the OpenAI `audio/transcriptions` API. For details on the available
+            parameters, see: <https://platform.openai.com/docs/api-reference/audio/createTranscription>
 
     Returns:
         A dictionary containing the transcription and other metadata.
@@ -264,27 +328,16 @@ async def transcriptions(
 
         >>> tbl.add_computed_column(transcription=transcriptions(tbl.audio, model='whisper-1', language='en'))
     """
+    if model_kwargs is None:
+        model_kwargs = {}
+
     file = pathlib.Path(audio)
-    transcription = await _openai_client().audio.transcriptions.create(
-        file=file,
-        model=model,
-        language=_opt(language),
-        prompt=_opt(prompt),
-        temperature=_opt(temperature),
-        timeout=_opt(timeout),
-    )
+    transcription = await _openai_client().audio.transcriptions.create(file=file, model=model, **model_kwargs)
     return transcription.dict()
 
 
 @pxt.udf
-async def translations(
-    audio: pxt.Audio,
-    *,
-    model: str,
-    prompt: Optional[str] = None,
-    temperature: Optional[float] = None,
-    timeout: Optional[float] = None,
-) -> dict:
+async def translations(audio: pxt.Audio, *, model: str, model_kwargs: dict[str, Any] | None = None) -> dict:
     """
     Translates audio into English.
 
@@ -302,8 +355,8 @@ async def translations(
     Args:
         audio: The audio to translate.
         model: The model to use for speech transcription and translation.
-
-    For details on the other parameters, see: <https://platform.openai.com/docs/api-reference/audio/createTranslation>
+        model_kwargs: Additional keyword args for the OpenAI `audio/translations` API. For details on the available
+            parameters, see: <https://platform.openai.com/docs/api-reference/audio/createTranslation>
 
     Returns:
         A dictionary containing the translation and other metadata.
@@ -314,10 +367,11 @@ async def translations(
 
         >>> tbl.add_computed_column(translation=translations(tbl.audio, model='whisper-1', language='en'))
     """
+    if model_kwargs is None:
+        model_kwargs = {}
+
     file = pathlib.Path(audio)
-    translation = await _openai_client().audio.translations.create(
-        file=file, model=model, prompt=_opt(prompt), temperature=_opt(temperature), timeout=_opt(timeout)
-    )
+    translation = await _openai_client().audio.translations.create(file=file, model=model, **model_kwargs)
     return translation.dict()
 
 
@@ -353,8 +407,15 @@ def _is_model_family(model: str, family: str) -> bool:
 
 
 def _chat_completions_get_request_resources(
-    messages: list, model: str, max_completion_tokens: Optional[int], max_tokens: Optional[int], n: Optional[int]
+    messages: list, model: str, model_kwargs: dict[str, Any] | None
 ) -> dict[str, int]:
+    if model_kwargs is None:
+        model_kwargs = {}
+
+    max_completion_tokens = model_kwargs.get('max_completion_tokens')
+    max_tokens = model_kwargs.get('max_tokens')
+    n = model_kwargs.get('n')
+
     completion_tokens = (n or 1) * (max_completion_tokens or max_tokens or _default_max_tokens(model))
 
     num_tokens = 0.0
@@ -373,24 +434,10 @@ async def chat_completions(
     messages: list,
     *,
     model: str,
-    frequency_penalty: Optional[float] = None,
-    logit_bias: Optional[dict[str, int]] = None,
-    logprobs: Optional[bool] = None,
-    top_logprobs: Optional[int] = None,
-    max_completion_tokens: Optional[int] = None,
-    max_tokens: Optional[int] = None,
-    n: Optional[int] = None,
-    presence_penalty: Optional[float] = None,
-    reasoning_effort: Optional[Literal['low', 'medium', 'high']] = None,
-    response_format: Optional[dict] = None,
-    seed: Optional[int] = None,
-    stop: Optional[list[str]] = None,
-    temperature: Optional[float] = None,
-    tools: Optional[list[dict]] = None,
-    tool_choice: Optional[dict] = None,
-    top_p: Optional[float] = None,
-    user: Optional[str] = None,
-    timeout: Optional[float] = None,
+    model_kwargs: dict[str, Any] | None = None,
+    tools: list[dict[str, Any]] | None = None,
+    tool_choice: dict[str, Any] | None = None,
+    _runtime_ctx: env.RuntimeCtx | None = None,
 ) -> dict:
     """
     Creates a model response for the given chat conversation.
@@ -409,8 +456,8 @@ async def chat_completions(
     Args:
         messages: A list of messages to use for chat completion, as described in the OpenAI API documentation.
         model: The model to use for chat completion.
-
-    For details on the other parameters, see: <https://platform.openai.com/docs/api-reference/chat>
+        model_kwargs: Additional keyword args for the OpenAI `chat/completions` API. For details on the available
+            parameters, see: <https://platform.openai.com/docs/api-reference/chat/create>
 
     Returns:
         A dictionary containing the response and other metadata.
@@ -420,27 +467,28 @@ async def chat_completions(
         of the table `tbl`:
 
         >>> messages = [
-                {'role': 'system', 'content': 'You are a helpful assistant.'},
-                {'role': 'user', 'content': tbl.prompt}
-            ]
-            tbl.add_computed_column(response=chat_completions(messages, model='gpt-4o-mini'))
+        ...     {'role': 'system', 'content': 'You are a helpful assistant.'},
+        ...     {'role': 'user', 'content': tbl.prompt}
+        ... ]
+        >>> tbl.add_computed_column(response=chat_completions(messages, model='gpt-4o-mini'))
     """
+    if model_kwargs is None:
+        model_kwargs = {}
+
     if tools is not None:
-        tools = [{'type': 'function', 'function': tool} for tool in tools]
+        model_kwargs['tools'] = [{'type': 'function', 'function': tool} for tool in tools]
 
-    tool_choice_: Union[str, dict, None] = None
     if tool_choice is not None:
         if tool_choice['auto']:
-            tool_choice_ = 'auto'
+            model_kwargs['tool_choice'] = 'auto'
         elif tool_choice['required']:
-            tool_choice_ = 'required'
+            model_kwargs['tool_choice'] = 'required'
         else:
             assert tool_choice['tool'] is not None
-            tool_choice_ = {'type': 'function', 'function': {'name': tool_choice['tool']}}
+            model_kwargs['tool_choice'] = {'type': 'function', 'function': {'name': tool_choice['tool']}}
 
-    extra_body: Optional[dict[str, Any]] = None
     if tool_choice is not None and not tool_choice['parallel_tool_calls']:
-        extra_body = {'parallel_tool_calls': False}
+        model_kwargs['parallel_tool_calls'] = False
 
     # make sure the pool info exists prior to making the request
     resource_pool = _rate_limits_pool(model)
@@ -448,45 +496,28 @@ async def chat_completions(
         resource_pool, lambda: OpenAIRateLimitsInfo(_chat_completions_get_request_resources)
     )
 
-    # cast(Any, ...): avoid mypy errors
+    request_ts = datetime.datetime.now(tz=datetime.timezone.utc)
     result = await _openai_client().chat.completions.with_raw_response.create(
-        messages=messages,
-        model=model,
-        frequency_penalty=_opt(frequency_penalty),
-        logit_bias=_opt(logit_bias),
-        logprobs=_opt(logprobs),
-        top_logprobs=_opt(top_logprobs),
-        max_completion_tokens=_opt(max_completion_tokens),
-        max_tokens=_opt(max_tokens),
-        n=_opt(n),
-        presence_penalty=_opt(presence_penalty),
-        reasoning_effort=_opt(reasoning_effort),
-        response_format=_opt(cast(Any, response_format)),
-        seed=_opt(seed),
-        stop=_opt(stop),
-        temperature=_opt(temperature),
-        tools=_opt(cast(Any, tools)),
-        tool_choice=_opt(cast(Any, tool_choice_)),
-        top_p=_opt(top_p),
-        user=_opt(user),
-        timeout=_opt(timeout),
-        extra_body=extra_body,
+        messages=messages, model=model, **model_kwargs
     )
 
     requests_info, tokens_info = _get_header_info(result.headers)
-    rate_limits_info.record(requests=requests_info, tokens=tokens_info)
+    is_retry = _runtime_ctx is not None and _runtime_ctx.is_retry
+    rate_limits_info.record(request_ts=request_ts, requests=requests_info, tokens=tokens_info, reset_exc=is_retry)
 
     return json.loads(result.text)
 
 
 def _vision_get_request_resources(
-    prompt: str,
-    image: PIL.Image.Image,
-    model: str,
-    max_completion_tokens: Optional[int],
-    max_tokens: Optional[int],
-    n: Optional[int],
+    prompt: str, image: PIL.Image.Image, model: str, model_kwargs: dict[str, Any] | None = None
 ) -> dict[str, int]:
+    if model_kwargs is None:
+        model_kwargs = {}
+
+    max_completion_tokens = model_kwargs.get('max_completion_tokens')
+    max_tokens = model_kwargs.get('max_tokens')
+    n = model_kwargs.get('n')
+
     completion_tokens = (n or 1) * (max_completion_tokens or max_tokens or _default_max_tokens(model))
     prompt_tokens = len(prompt) / 4
 
@@ -519,10 +550,8 @@ async def vision(
     image: PIL.Image.Image,
     *,
     model: str,
-    max_completion_tokens: Optional[int] = None,
-    max_tokens: Optional[int] = None,
-    n: Optional[int] = 1,
-    timeout: Optional[float] = None,
+    model_kwargs: dict[str, Any] | None = None,
+    _runtime_ctx: env.RuntimeCtx | None = None,
 ) -> str:
     """
     Analyzes an image with the OpenAI vision capability. This is a convenience function that takes an image and
@@ -552,6 +581,9 @@ async def vision(
 
         >>> tbl.add_computed_column(response=vision("What's in this image?", tbl.image, model='gpt-4o-mini'))
     """
+    if model_kwargs is None:
+        model_kwargs = {}
+
     # TODO(aaron-siegel): Decompose CPU/GPU ops into separate functions
     bytes_arr = io.BytesIO()
     image.save(bytes_arr, format='png')
@@ -573,17 +605,17 @@ async def vision(
         resource_pool, lambda: OpenAIRateLimitsInfo(_vision_get_request_resources)
     )
 
+    request_ts = datetime.datetime.now(tz=datetime.timezone.utc)
     result = await _openai_client().chat.completions.with_raw_response.create(
         messages=messages,  # type: ignore
         model=model,
-        max_completion_tokens=_opt(max_completion_tokens),
-        max_tokens=_opt(max_tokens),
-        n=_opt(n),
-        timeout=_opt(timeout),
+        **model_kwargs,
     )
 
+    # _logger.debug(f'vision(): headers={result.headers}')
     requests_info, tokens_info = _get_header_info(result.headers)
-    rate_limits_info.record(requests=requests_info, tokens=tokens_info)
+    is_retry = _runtime_ctx is not None and _runtime_ctx.is_retry
+    rate_limits_info.record(request_ts=request_ts, requests=requests_info, tokens=tokens_info, reset_exc=is_retry)
 
     result = json.loads(result.text)
     return result['choices'][0]['message']['content']
@@ -609,9 +641,8 @@ async def embeddings(
     input: Batch[str],
     *,
     model: str,
-    dimensions: Optional[int] = None,
-    user: Optional[str] = None,
-    timeout: Optional[float] = None,
+    model_kwargs: dict[str, Any] | None = None,
+    _runtime_ctx: env.RuntimeCtx | None = None,
 ) -> Batch[pxt.Array[(None,), pxt.Float]]:
     """
     Creates an embedding vector representing the input text.
@@ -630,10 +661,8 @@ async def embeddings(
     Args:
         input: The text to embed.
         model: The model to use for the embedding.
-        dimensions: The vector length of the embedding. If not specified, Pixeltable will use
-            a default value based on the model.
-
-    For details on the other parameters, see: <https://platform.openai.com/docs/api-reference/embeddings>
+        model_kwargs: Additional keyword args for the OpenAI `embeddings` API. For details on the available
+            parameters, see: <https://platform.openai.com/docs/api-reference/embeddings>
 
     Returns:
         An array representing the application of the given embedding to `input`.
@@ -648,26 +677,29 @@ async def embeddings(
 
         >>> tbl.add_embedding_index(embedding=embeddings.using(model='text-embedding-3-small'))
     """
+    if model_kwargs is None:
+        model_kwargs = {}
+
     _logger.debug(f'embeddings: batch_size={len(input)}')
     resource_pool = _rate_limits_pool(model)
     rate_limits_info = env.Env.get().get_resource_pool_info(
         resource_pool, lambda: OpenAIRateLimitsInfo(_embeddings_get_request_resources)
     )
+    request_ts = datetime.datetime.now(tz=datetime.timezone.utc)
     result = await _openai_client().embeddings.with_raw_response.create(
-        input=input,
-        model=model,
-        dimensions=_opt(dimensions),
-        user=_opt(user),
-        encoding_format='float',
-        timeout=_opt(timeout),
+        input=input, model=model, encoding_format='float', **model_kwargs
     )
     requests_info, tokens_info = _get_header_info(result.headers)
-    rate_limits_info.record(requests=requests_info, tokens=tokens_info)
+    is_retry = _runtime_ctx is not None and _runtime_ctx.is_retry
+    rate_limits_info.record(request_ts=request_ts, requests=requests_info, tokens=tokens_info, reset_exc=is_retry)
     return [np.array(data['embedding'], dtype=np.float64) for data in json.loads(result.content)['data']]
 
 
 @embeddings.conditional_return_type
-def _(model: str, dimensions: Optional[int] = None) -> ts.ArrayType:
+def _(model: str, model_kwargs: dict[str, Any] | None = None) -> ts.ArrayType:
+    dimensions: int | None = None
+    if model_kwargs is not None:
+        dimensions = model_kwargs.get('dimensions')
     if dimensions is None:
         if model not in _embedding_dimensions_cache:
             # TODO: find some other way to retrieve a sample
@@ -682,14 +714,7 @@ def _(model: str, dimensions: Optional[int] = None) -> ts.ArrayType:
 
 @pxt.udf
 async def image_generations(
-    prompt: str,
-    *,
-    model: str = 'dall-e-2',
-    quality: Optional[str] = None,
-    size: Optional[str] = None,
-    style: Optional[str] = None,
-    user: Optional[str] = None,
-    timeout: Optional[float] = None,
+    prompt: str, *, model: str = 'dall-e-2', model_kwargs: dict[str, Any] | None = None
 ) -> PIL.Image.Image:
     """
     Creates an image given a prompt.
@@ -708,8 +733,8 @@ async def image_generations(
     Args:
         prompt: Prompt for the image.
         model: The model to use for the generations.
-
-    For details on the other parameters, see: <https://platform.openai.com/docs/api-reference/images/create>
+        model_kwargs: Additional keyword args for the OpenAI `images/generations` API. For details on the available
+            parameters, see: <https://platform.openai.com/docs/api-reference/images/create>
 
     Returns:
         The generated image.
@@ -720,16 +745,12 @@ async def image_generations(
 
         >>> tbl.add_computed_column(gen_image=image_generations(tbl.text, model='dall-e-2'))
     """
+    if model_kwargs is None:
+        model_kwargs = {}
+
     # TODO(aaron-siegel): Decompose CPU/GPU ops into separate functions
     result = await _openai_client().images.generate(
-        prompt=prompt,
-        model=_opt(model),
-        quality=_opt(quality),  # type: ignore
-        size=_opt(size),  # type: ignore
-        style=_opt(style),  # type: ignore
-        user=_opt(user),
-        response_format='b64_json',
-        timeout=_opt(timeout),
+        prompt=prompt, model=model, response_format='b64_json', **model_kwargs
     )
     b64_str = result.data[0].b64_json
     b64_bytes = base64.b64decode(b64_str)
@@ -739,9 +760,11 @@ async def image_generations(
 
 
 @image_generations.conditional_return_type
-def _(size: Optional[str] = None) -> ts.ImageType:
-    if size is None:
+def _(model_kwargs: dict[str, Any] | None = None) -> ts.ImageType:
+    if model_kwargs is None or 'size' not in model_kwargs:
+        # default size is 1024x1024
         return ts.ImageType(size=(1024, 1024))
+    size = model_kwargs['size']
     x_pos = size.find('x')
     if x_pos == -1:
         return ts.ImageType()
@@ -787,7 +810,7 @@ async def moderations(input: str, *, model: str = 'omni-moderation-latest') -> d
 
         >>> tbl.add_computed_column(moderations=moderations(tbl.text, model='text-moderation-stable'))
     """
-    result = await _openai_client().moderations.create(input=input, model=_opt(model))
+    result = await _openai_client().moderations.create(input=input, model=model)
     return result.dict()
 
 
@@ -813,7 +836,7 @@ def invoke_tools(tools: Tools, response: exprs.Expr) -> exprs.InlineDict:
 
 
 @pxt.udf
-def _openai_response_to_pxt_tool_calls(response: dict) -> Optional[dict]:
+def _openai_response_to_pxt_tool_calls(response: dict) -> dict | None:
     if 'tool_calls' not in response['choices'][0]['message'] or response['choices'][0]['message']['tool_calls'] is None:
         return None
     openai_tool_calls = response['choices'][0]['message']['tool_calls']
@@ -826,15 +849,6 @@ def _openai_response_to_pxt_tool_calls(response: dict) -> Optional[dict]:
     return pxt_tool_calls
 
 
-_T = TypeVar('_T')
-
-
-def _opt(arg: _T) -> Union[_T, 'openai.NotGiven']:
-    import openai
-
-    return arg if arg is not None else openai.NOT_GIVEN
-
-
 __all__ = local_public_names(__name__)