gllm-inference-binary 0.5.65__cp313-cp313-macosx_13_0_arm64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- gllm_inference/__init__.pyi +0 -0
- gllm_inference/builder/__init__.pyi +6 -0
- gllm_inference/builder/_build_invoker.pyi +28 -0
- gllm_inference/builder/build_em_invoker.pyi +130 -0
- gllm_inference/builder/build_lm_invoker.pyi +222 -0
- gllm_inference/builder/build_lm_request_processor.pyi +88 -0
- gllm_inference/builder/build_output_parser.pyi +29 -0
- gllm_inference/catalog/__init__.pyi +4 -0
- gllm_inference/catalog/catalog.pyi +121 -0
- gllm_inference/catalog/lm_request_processor_catalog.pyi +112 -0
- gllm_inference/catalog/prompt_builder_catalog.pyi +82 -0
- gllm_inference/constants.pyi +12 -0
- gllm_inference/em_invoker/__init__.pyi +12 -0
- gllm_inference/em_invoker/azure_openai_em_invoker.pyi +88 -0
- gllm_inference/em_invoker/bedrock_em_invoker.pyi +118 -0
- gllm_inference/em_invoker/cohere_em_invoker.pyi +127 -0
- gllm_inference/em_invoker/em_invoker.pyi +90 -0
- gllm_inference/em_invoker/google_em_invoker.pyi +129 -0
- gllm_inference/em_invoker/jina_em_invoker.pyi +103 -0
- gllm_inference/em_invoker/langchain/__init__.pyi +3 -0
- gllm_inference/em_invoker/langchain/em_invoker_embeddings.pyi +84 -0
- gllm_inference/em_invoker/langchain_em_invoker.pyi +46 -0
- gllm_inference/em_invoker/openai_compatible_em_invoker.pyi +41 -0
- gllm_inference/em_invoker/openai_em_invoker.pyi +118 -0
- gllm_inference/em_invoker/schema/__init__.pyi +0 -0
- gllm_inference/em_invoker/schema/bedrock.pyi +29 -0
- gllm_inference/em_invoker/schema/cohere.pyi +20 -0
- gllm_inference/em_invoker/schema/google.pyi +9 -0
- gllm_inference/em_invoker/schema/jina.pyi +29 -0
- gllm_inference/em_invoker/schema/langchain.pyi +5 -0
- gllm_inference/em_invoker/schema/openai.pyi +7 -0
- gllm_inference/em_invoker/schema/openai_compatible.pyi +7 -0
- gllm_inference/em_invoker/schema/twelvelabs.pyi +17 -0
- gllm_inference/em_invoker/schema/voyage.pyi +15 -0
- gllm_inference/em_invoker/twelevelabs_em_invoker.pyi +101 -0
- gllm_inference/em_invoker/voyage_em_invoker.pyi +104 -0
- gllm_inference/exceptions/__init__.pyi +4 -0
- gllm_inference/exceptions/error_parser.pyi +41 -0
- gllm_inference/exceptions/exceptions.pyi +132 -0
- gllm_inference/exceptions/provider_error_map.pyi +24 -0
- gllm_inference/lm_invoker/__init__.pyi +15 -0
- gllm_inference/lm_invoker/anthropic_lm_invoker.pyi +320 -0
- gllm_inference/lm_invoker/azure_openai_lm_invoker.pyi +237 -0
- gllm_inference/lm_invoker/batch/__init__.pyi +3 -0
- gllm_inference/lm_invoker/batch/batch_operations.pyi +128 -0
- gllm_inference/lm_invoker/bedrock_lm_invoker.pyi +212 -0
- gllm_inference/lm_invoker/datasaur_lm_invoker.pyi +157 -0
- gllm_inference/lm_invoker/google_lm_invoker.pyi +421 -0
- gllm_inference/lm_invoker/langchain_lm_invoker.pyi +239 -0
- gllm_inference/lm_invoker/litellm_lm_invoker.pyi +224 -0
- gllm_inference/lm_invoker/lm_invoker.pyi +183 -0
- gllm_inference/lm_invoker/openai_chat_completions_lm_invoker.pyi +252 -0
- gllm_inference/lm_invoker/openai_compatible_lm_invoker.pyi +52 -0
- gllm_inference/lm_invoker/openai_lm_invoker.pyi +437 -0
- gllm_inference/lm_invoker/portkey_lm_invoker.pyi +296 -0
- gllm_inference/lm_invoker/schema/__init__.pyi +0 -0
- gllm_inference/lm_invoker/schema/anthropic.pyi +56 -0
- gllm_inference/lm_invoker/schema/bedrock.pyi +53 -0
- gllm_inference/lm_invoker/schema/datasaur.pyi +14 -0
- gllm_inference/lm_invoker/schema/google.pyi +36 -0
- gllm_inference/lm_invoker/schema/langchain.pyi +23 -0
- gllm_inference/lm_invoker/schema/openai.pyi +109 -0
- gllm_inference/lm_invoker/schema/openai_chat_completions.pyi +62 -0
- gllm_inference/lm_invoker/schema/portkey.pyi +31 -0
- gllm_inference/lm_invoker/schema/xai.pyi +31 -0
- gllm_inference/lm_invoker/sea_lion_lm_invoker.pyi +48 -0
- gllm_inference/lm_invoker/xai_lm_invoker.pyi +252 -0
- gllm_inference/model/__init__.pyi +13 -0
- gllm_inference/model/em/__init__.pyi +0 -0
- gllm_inference/model/em/cohere_em.pyi +17 -0
- gllm_inference/model/em/google_em.pyi +16 -0
- gllm_inference/model/em/jina_em.pyi +22 -0
- gllm_inference/model/em/openai_em.pyi +15 -0
- gllm_inference/model/em/twelvelabs_em.pyi +13 -0
- gllm_inference/model/em/voyage_em.pyi +20 -0
- gllm_inference/model/lm/__init__.pyi +0 -0
- gllm_inference/model/lm/anthropic_lm.pyi +22 -0
- gllm_inference/model/lm/google_lm.pyi +18 -0
- gllm_inference/model/lm/openai_lm.pyi +27 -0
- gllm_inference/model/lm/sea_lion_lm.pyi +16 -0
- gllm_inference/model/lm/xai_lm.pyi +19 -0
- gllm_inference/output_parser/__init__.pyi +3 -0
- gllm_inference/output_parser/json_output_parser.pyi +60 -0
- gllm_inference/output_parser/output_parser.pyi +27 -0
- gllm_inference/prompt_builder/__init__.pyi +3 -0
- gllm_inference/prompt_builder/format_strategy/__init__.pyi +4 -0
- gllm_inference/prompt_builder/format_strategy/format_strategy.pyi +55 -0
- gllm_inference/prompt_builder/format_strategy/jinja_format_strategy.pyi +45 -0
- gllm_inference/prompt_builder/format_strategy/string_format_strategy.pyi +20 -0
- gllm_inference/prompt_builder/prompt_builder.pyi +73 -0
- gllm_inference/prompt_formatter/__init__.pyi +7 -0
- gllm_inference/prompt_formatter/agnostic_prompt_formatter.pyi +49 -0
- gllm_inference/prompt_formatter/huggingface_prompt_formatter.pyi +55 -0
- gllm_inference/prompt_formatter/llama_prompt_formatter.pyi +59 -0
- gllm_inference/prompt_formatter/mistral_prompt_formatter.pyi +53 -0
- gllm_inference/prompt_formatter/openai_prompt_formatter.pyi +35 -0
- gllm_inference/prompt_formatter/prompt_formatter.pyi +30 -0
- gllm_inference/realtime_chat/__init__.pyi +3 -0
- gllm_inference/realtime_chat/google_realtime_chat.pyi +205 -0
- gllm_inference/realtime_chat/input_streamer/__init__.pyi +4 -0
- gllm_inference/realtime_chat/input_streamer/input_streamer.pyi +36 -0
- gllm_inference/realtime_chat/input_streamer/keyboard_input_streamer.pyi +27 -0
- gllm_inference/realtime_chat/input_streamer/linux_mic_input_streamer.pyi +36 -0
- gllm_inference/realtime_chat/output_streamer/__init__.pyi +4 -0
- gllm_inference/realtime_chat/output_streamer/console_output_streamer.pyi +21 -0
- gllm_inference/realtime_chat/output_streamer/linux_speaker_output_streamer.pyi +42 -0
- gllm_inference/realtime_chat/output_streamer/output_streamer.pyi +33 -0
- gllm_inference/realtime_chat/realtime_chat.pyi +28 -0
- gllm_inference/request_processor/__init__.pyi +4 -0
- gllm_inference/request_processor/lm_request_processor.pyi +101 -0
- gllm_inference/request_processor/uses_lm_mixin.pyi +130 -0
- gllm_inference/schema/__init__.pyi +19 -0
- gllm_inference/schema/activity.pyi +64 -0
- gllm_inference/schema/attachment.pyi +102 -0
- gllm_inference/schema/code_exec_result.pyi +14 -0
- gllm_inference/schema/config.pyi +15 -0
- gllm_inference/schema/enums.pyi +82 -0
- gllm_inference/schema/events.pyi +105 -0
- gllm_inference/schema/formatter.pyi +31 -0
- gllm_inference/schema/lm_input.pyi +4 -0
- gllm_inference/schema/lm_output.pyi +266 -0
- gllm_inference/schema/mcp.pyi +31 -0
- gllm_inference/schema/message.pyi +52 -0
- gllm_inference/schema/model_id.pyi +176 -0
- gllm_inference/schema/reasoning.pyi +15 -0
- gllm_inference/schema/token_usage.pyi +75 -0
- gllm_inference/schema/tool_call.pyi +14 -0
- gllm_inference/schema/tool_result.pyi +11 -0
- gllm_inference/schema/type_alias.pyi +11 -0
- gllm_inference/utils/__init__.pyi +5 -0
- gllm_inference/utils/io_utils.pyi +26 -0
- gllm_inference/utils/langchain.pyi +30 -0
- gllm_inference/utils/validation.pyi +15 -0
- gllm_inference.build/.gitignore +1 -0
- gllm_inference.cpython-313-darwin.so +0 -0
- gllm_inference.pyi +156 -0
- gllm_inference_binary-0.5.65.dist-info/METADATA +138 -0
- gllm_inference_binary-0.5.65.dist-info/RECORD +140 -0
- gllm_inference_binary-0.5.65.dist-info/WHEEL +5 -0
- gllm_inference_binary-0.5.65.dist-info/top_level.txt +1 -0
|
File without changes
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
from gllm_inference.builder.build_em_invoker import build_em_invoker as build_em_invoker
|
|
2
|
+
from gllm_inference.builder.build_lm_invoker import build_lm_invoker as build_lm_invoker
|
|
3
|
+
from gllm_inference.builder.build_lm_request_processor import build_lm_request_processor as build_lm_request_processor
|
|
4
|
+
from gllm_inference.builder.build_output_parser import build_output_parser as build_output_parser
|
|
5
|
+
|
|
6
|
+
__all__ = ['build_em_invoker', 'build_lm_invoker', 'build_lm_request_processor', 'build_output_parser']
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
from _typeshed import Incomplete
|
|
2
|
+
from gllm_inference.em_invoker.em_invoker import BaseEMInvoker as BaseEMInvoker
|
|
3
|
+
from gllm_inference.lm_invoker.lm_invoker import BaseLMInvoker as BaseLMInvoker
|
|
4
|
+
from gllm_inference.schema.model_id import ModelId as ModelId, ModelProvider as ModelProvider, PROVIDERS_OPTIONAL_PATH as PROVIDERS_OPTIONAL_PATH
|
|
5
|
+
|
|
6
|
+
logger: Incomplete
|
|
7
|
+
|
|
8
|
+
class Key:
|
|
9
|
+
"""Defines valid keys in the config."""
|
|
10
|
+
ACCESS_KEY_ID: str
|
|
11
|
+
API_KEY: str
|
|
12
|
+
AZURE_DEPLOYMENT: str
|
|
13
|
+
AZURE_ENDPOINT: str
|
|
14
|
+
BASE_URL: str
|
|
15
|
+
CONFIG: str
|
|
16
|
+
CUSTOM_HOST: str
|
|
17
|
+
CREDENTIALS_PATH: str
|
|
18
|
+
MODEL_ID: str
|
|
19
|
+
MODEL_KWARGS: str
|
|
20
|
+
MODEL_NAME: str
|
|
21
|
+
MODEL_CLASS_PATH: str
|
|
22
|
+
PORTKEY_API_KEY: str
|
|
23
|
+
PROVIDER: str
|
|
24
|
+
SECRET_ACCESS_KEY: str
|
|
25
|
+
|
|
26
|
+
PROVIDERS_REQUIRE_BASE_URL: Incomplete
|
|
27
|
+
MODEL_NAME_KEY_MAP: Incomplete
|
|
28
|
+
DEFAULT_MODEL_NAME_KEY: Incomplete
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
from gllm_inference.em_invoker import AzureOpenAIEMInvoker as AzureOpenAIEMInvoker, BedrockEMInvoker as BedrockEMInvoker, CohereEMInvoker as CohereEMInvoker, GoogleEMInvoker as GoogleEMInvoker, JinaEMInvoker as JinaEMInvoker, LangChainEMInvoker as LangChainEMInvoker, OpenAICompatibleEMInvoker as OpenAICompatibleEMInvoker, OpenAIEMInvoker as OpenAIEMInvoker, TwelveLabsEMInvoker as TwelveLabsEMInvoker, VoyageEMInvoker as VoyageEMInvoker
|
|
2
|
+
from gllm_inference.em_invoker.em_invoker import BaseEMInvoker as BaseEMInvoker
|
|
3
|
+
from gllm_inference.schema.model_id import ModelId as ModelId, ModelProvider as ModelProvider
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
PROVIDER_TO_EM_INVOKER_MAP: dict[str, type[BaseEMInvoker]]
|
|
7
|
+
|
|
8
|
+
def build_em_invoker(model_id: str | ModelId, credentials: str | dict[str, Any] | None = None, config: dict[str, Any] | None = None) -> BaseEMInvoker:
|
|
9
|
+
'''Build an embedding model invoker based on the provided configurations.
|
|
10
|
+
|
|
11
|
+
Args:
|
|
12
|
+
model_id (str | ModelId): The model id, can either be a ModelId instance or a string in a format defined
|
|
13
|
+
in the following page: https://gdplabs.gitbook.io/sdk/resources/supported-models#embedding-models-ems
|
|
14
|
+
credentials (str | dict[str, Any] | None, optional): The credentials for the language model. Can either be:
|
|
15
|
+
1. An API key.
|
|
16
|
+
2. A path to a credentials JSON file, currently only supported for Google Vertex AI.
|
|
17
|
+
3. A dictionary of credentials, currently only supported for LangChain.
|
|
18
|
+
Defaults to None, in which case the credentials will be loaded from the appropriate environment variables.
|
|
19
|
+
config (dict[str, Any] | None, optional): Additional configuration for the embedding model. Defaults to None.
|
|
20
|
+
|
|
21
|
+
Returns:
|
|
22
|
+
BaseEMInvoker: The initialized embedding model invoker.
|
|
23
|
+
|
|
24
|
+
Raises:
|
|
25
|
+
ValueError: If the provider is invalid.
|
|
26
|
+
|
|
27
|
+
Usage examples:
|
|
28
|
+
# Using Bedrock
|
|
29
|
+
```python
|
|
30
|
+
em_invoker = build_em_invoker(
|
|
31
|
+
model_id="bedrock/cohere.embed-english-v3",
|
|
32
|
+
credentials={
|
|
33
|
+
"access_key_id": "Abc123...",
|
|
34
|
+
"secret_access_key": "Xyz123...",
|
|
35
|
+
},
|
|
36
|
+
)
|
|
37
|
+
```
|
|
38
|
+
The credentials can also be provided through the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`
|
|
39
|
+
environment variables.
|
|
40
|
+
|
|
41
|
+
# Using Google Gen AI (via API key)
|
|
42
|
+
```python
|
|
43
|
+
em_invoker = build_em_invoker(
|
|
44
|
+
model_id="google/text-embedding-004",
|
|
45
|
+
credentials="AIzaSyD..."
|
|
46
|
+
)
|
|
47
|
+
```
|
|
48
|
+
The credentials can also be provided through the `GOOGLE_API_KEY` environment variable.
|
|
49
|
+
|
|
50
|
+
# Using Google Vertex AI (via service account)
|
|
51
|
+
```python
|
|
52
|
+
em_invoker = build_em_invoker(
|
|
53
|
+
model_id="google/text-embedding-004",
|
|
54
|
+
credentials="/path/to/google-credentials.json"
|
|
55
|
+
)
|
|
56
|
+
```
|
|
57
|
+
Providing credentials through environment variable is not supported for Google Vertex AI.
|
|
58
|
+
|
|
59
|
+
# Using Jina
|
|
60
|
+
```python
|
|
61
|
+
em_invoker = build_em_invoker(
|
|
62
|
+
model_id="jina/jina-embeddings-v2-large",
|
|
63
|
+
credentials="jina-api-key"
|
|
64
|
+
)
|
|
65
|
+
```
|
|
66
|
+
The credentials can also be provided through the `JINA_API_KEY` environment variable. For the list of supported
|
|
67
|
+
models, please refer to the following page: https://jina.ai/models
|
|
68
|
+
|
|
69
|
+
# Using OpenAI
|
|
70
|
+
```python
|
|
71
|
+
em_invoker = build_em_invoker(
|
|
72
|
+
model_id="openai/text-embedding-3-small",
|
|
73
|
+
credentials="sk-..."
|
|
74
|
+
)
|
|
75
|
+
```
|
|
76
|
+
The credentials can also be provided through the `OPENAI_API_KEY` environment variable.
|
|
77
|
+
|
|
78
|
+
# Using OpenAI Embeddings API-compatible endpoints (e.g. vLLM)
|
|
79
|
+
```python
|
|
80
|
+
em_invoker = build_em_invoker(
|
|
81
|
+
model_id="openai/https://my-vllm-url:8000/v1:my-model-name",
|
|
82
|
+
credentials="sk-..."
|
|
83
|
+
)
|
|
84
|
+
```
|
|
85
|
+
The credentials can also be provided through the `OPENAI_API_KEY` environment variable.
|
|
86
|
+
|
|
87
|
+
# Using Azure OpenAI
|
|
88
|
+
```python
|
|
89
|
+
em_invoker = build_em_invoker(
|
|
90
|
+
model_id="azure-openai/https://my-resource.openai.azure.com/openai/v1:my-deployment",
|
|
91
|
+
credentials="azure-api-key"
|
|
92
|
+
)
|
|
93
|
+
```
|
|
94
|
+
The credentials can also be provided through the `AZURE_OPENAI_API_KEY` environment variable.
|
|
95
|
+
|
|
96
|
+
# Using TwelveLabs
|
|
97
|
+
```python
|
|
98
|
+
em_invoker = build_em_invoker(
|
|
99
|
+
model_id="twelvelabs/Marengo-retrieval-2.7",
|
|
100
|
+
credentials="tlk_..."
|
|
101
|
+
)
|
|
102
|
+
```
|
|
103
|
+
The credentials can also be provided through the `TWELVELABS_API_KEY` environment variable.
|
|
104
|
+
|
|
105
|
+
# Using Voyage
|
|
106
|
+
```python
|
|
107
|
+
em_invoker = build_em_invoker(
|
|
108
|
+
model_id="voyage/voyage-3.5-lite",
|
|
109
|
+
credentials="sk-..."
|
|
110
|
+
)
|
|
111
|
+
```
|
|
112
|
+
The credentials can also be provided through the `VOYAGE_API_KEY` environment variable.
|
|
113
|
+
|
|
114
|
+
# Using LangChain
|
|
115
|
+
```python
|
|
116
|
+
em_invoker = build_em_invoker(
|
|
117
|
+
model_id="langchain/langchain_openai.OpenAIEmbeddings:text-embedding-3-small",
|
|
118
|
+
credentials={"api_key": "sk-..."}
|
|
119
|
+
)
|
|
120
|
+
```
|
|
121
|
+
The credentials can also be provided through various environment variables depending on the
|
|
122
|
+
LangChain module being used. For the list of supported providers and the supported environment
|
|
123
|
+
variables credentials, please refer to the following page:
|
|
124
|
+
https://python.langchain.com/docs/integrations/text_embedding/
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
Security warning:
|
|
128
|
+
Please provide the EM invoker credentials ONLY to the `credentials` parameter. Do not put any kind of
|
|
129
|
+
credentials in the `config` parameter as the content of the `config` parameter will be logged.
|
|
130
|
+
'''
|
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
from gllm_inference.lm_invoker import AnthropicLMInvoker as AnthropicLMInvoker, AzureOpenAILMInvoker as AzureOpenAILMInvoker, BedrockLMInvoker as BedrockLMInvoker, DatasaurLMInvoker as DatasaurLMInvoker, GoogleLMInvoker as GoogleLMInvoker, LangChainLMInvoker as LangChainLMInvoker, LiteLLMLMInvoker as LiteLLMLMInvoker, OpenAIChatCompletionsLMInvoker as OpenAIChatCompletionsLMInvoker, OpenAICompatibleLMInvoker as OpenAICompatibleLMInvoker, OpenAILMInvoker as OpenAILMInvoker, PortkeyLMInvoker as PortkeyLMInvoker, SeaLionLMInvoker as SeaLionLMInvoker, XAILMInvoker as XAILMInvoker
|
|
2
|
+
from gllm_inference.lm_invoker.lm_invoker import BaseLMInvoker as BaseLMInvoker
|
|
3
|
+
from gllm_inference.schema.model_id import ModelId as ModelId, ModelProvider as ModelProvider
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
PROVIDER_TO_LM_INVOKER_MAP: dict[str, type[BaseLMInvoker]]
|
|
7
|
+
|
|
8
|
+
def build_lm_invoker(model_id: str | ModelId, credentials: str | dict[str, Any] | None = None, config: dict[str, Any] | None = None) -> BaseLMInvoker:
|
|
9
|
+
'''Build a language model invoker based on the provided configurations.
|
|
10
|
+
|
|
11
|
+
Args:
|
|
12
|
+
model_id (str | ModelId): The model id, can either be a ModelId instance or a string in a format defined
|
|
13
|
+
in the following page: https://gdplabs.gitbook.io/sdk/resources/supported-models#language-models-lms
|
|
14
|
+
credentials (str | dict[str, Any] | None, optional): The credentials for the language model. Can either be:
|
|
15
|
+
1. An API key.
|
|
16
|
+
2. A path to a credentials JSON file, currently only supported for Google Vertex AI.
|
|
17
|
+
3. A dictionary of credentials, currently supported for Bedrock and LangChain.
|
|
18
|
+
Defaults to None, in which case the credentials will be loaded from the appropriate environment variables.
|
|
19
|
+
config (dict[str, Any] | None, optional): Additional configuration for the language model. Defaults to None.
|
|
20
|
+
|
|
21
|
+
Returns:
|
|
22
|
+
BaseLMInvoker: The initialized language model invoker.
|
|
23
|
+
|
|
24
|
+
Raises:
|
|
25
|
+
ValueError: If the provider is invalid.
|
|
26
|
+
|
|
27
|
+
Usage examples:
|
|
28
|
+
# Using Anthropic
|
|
29
|
+
```python
|
|
30
|
+
lm_invoker = build_lm_invoker(
|
|
31
|
+
model_id="anthropic/claude-3-5-sonnet-latest",
|
|
32
|
+
credentials="sk-ant-api03-..."
|
|
33
|
+
)
|
|
34
|
+
```
|
|
35
|
+
The credentials can also be provided through the `ANTHROPIC_API_KEY` environment variable.
|
|
36
|
+
|
|
37
|
+
# Using Bedrock
|
|
38
|
+
```python
|
|
39
|
+
lm_invoker = build_lm_invoker(
|
|
40
|
+
model_id="bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0",
|
|
41
|
+
credentials={
|
|
42
|
+
"access_key_id": "Abc123...",
|
|
43
|
+
"secret_access_key": "Xyz123...",
|
|
44
|
+
},
|
|
45
|
+
)
|
|
46
|
+
```
|
|
47
|
+
The credentials can also be provided through the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`
|
|
48
|
+
environment variables.
|
|
49
|
+
|
|
50
|
+
# Using Datasaur LLM Projects Deployment API
|
|
51
|
+
```python
|
|
52
|
+
lm_invoker = build_lm_invoker(
|
|
53
|
+
model_id="datasaur/https://deployment.datasaur.ai/api/deployment/teamId/deploymentId/",
|
|
54
|
+
credentials="..."
|
|
55
|
+
)
|
|
56
|
+
```
|
|
57
|
+
The credentials can also be provided through the `DATASAUR_API_KEY` environment variable.
|
|
58
|
+
|
|
59
|
+
# Using Google Gen AI (via API key)
|
|
60
|
+
```python
|
|
61
|
+
lm_invoker = build_lm_invoker(
|
|
62
|
+
model_id="google/gemini-2.5-flash-lite",
|
|
63
|
+
credentials="AIzaSyD..."
|
|
64
|
+
)
|
|
65
|
+
```
|
|
66
|
+
The credentials can also be provided through the `GOOGLE_API_KEY` environment variable.
|
|
67
|
+
|
|
68
|
+
# Using Google Vertex AI (via service account)
|
|
69
|
+
```python
|
|
70
|
+
lm_invoker = build_lm_invoker(
|
|
71
|
+
model_id="google/gemini-2.5-flash-lite",
|
|
72
|
+
credentials="/path/to/google-credentials.json"
|
|
73
|
+
)
|
|
74
|
+
```
|
|
75
|
+
Providing credentials through environment variable is not supported for Google Vertex AI.
|
|
76
|
+
|
|
77
|
+
# Using OpenAI
|
|
78
|
+
```python
|
|
79
|
+
lm_invoker = build_lm_invoker(
|
|
80
|
+
model_id="openai/gpt-5-nano",
|
|
81
|
+
credentials="sk-..."
|
|
82
|
+
)
|
|
83
|
+
```
|
|
84
|
+
The credentials can also be provided through the `OPENAI_API_KEY` environment variable.
|
|
85
|
+
|
|
86
|
+
# Using OpenAI with Chat Completions API
|
|
87
|
+
```python
|
|
88
|
+
lm_invoker = build_lm_invoker(
|
|
89
|
+
model_id="openai-chat-completions/gpt-5-nano",
|
|
90
|
+
credentials="sk-..."
|
|
91
|
+
)
|
|
92
|
+
```
|
|
93
|
+
The credentials can also be provided through the `OPENAI_API_KEY` environment variable.
|
|
94
|
+
|
|
95
|
+
# Using OpenAI Responses API-compatible endpoints (e.g. SGLang)
|
|
96
|
+
```python
|
|
97
|
+
lm_invoker = build_lm_invoker(
|
|
98
|
+
model_id="openai/https://my-sglang-url:8000/v1:my-model-name",
|
|
99
|
+
credentials="sk-..."
|
|
100
|
+
)
|
|
101
|
+
```
|
|
102
|
+
The credentials can also be provided through the `OPENAI_API_KEY` environment variable.
|
|
103
|
+
|
|
104
|
+
# Using OpenAI Chat Completions API-compatible endpoints (e.g. Groq)
|
|
105
|
+
```python
|
|
106
|
+
lm_invoker = build_lm_invoker(
|
|
107
|
+
model_id="openai-chat-completions/https://api.groq.com/openai/v1:llama3-8b-8192",
|
|
108
|
+
credentials="gsk_..."
|
|
109
|
+
)
|
|
110
|
+
```
|
|
111
|
+
The credentials can also be provided through the `OPENAI_API_KEY` environment variable.
|
|
112
|
+
|
|
113
|
+
# Using Azure OpenAI
|
|
114
|
+
```python
|
|
115
|
+
lm_invoker = build_lm_invoker(
|
|
116
|
+
model_id="azure-openai/https://my-resource.openai.azure.com/openai/v1:my-deployment",
|
|
117
|
+
credentials="azure-api-key"
|
|
118
|
+
)
|
|
119
|
+
```
|
|
120
|
+
The credentials can also be provided through the `AZURE_OPENAI_API_KEY` environment variable.
|
|
121
|
+
|
|
122
|
+
# Using SEA-LION
|
|
123
|
+
```python
|
|
124
|
+
lm_invoker = build_lm_invoker(
|
|
125
|
+
model_id="sea-lion/aisingapore/Qwen-SEA-LION-v4-32B-IT",
|
|
126
|
+
credentials="sk-..."
|
|
127
|
+
)
|
|
128
|
+
```
|
|
129
|
+
The credentials can also be provided through the `SEA_LION_API_KEY` environment variable.
|
|
130
|
+
|
|
131
|
+
# Using LangChain
|
|
132
|
+
```python
|
|
133
|
+
lm_invoker = build_lm_invoker(
|
|
134
|
+
model_id="langchain/langchain_openai.ChatOpenAI:gpt-4o-mini",
|
|
135
|
+
credentials={"api_key": "sk-..."}
|
|
136
|
+
)
|
|
137
|
+
```
|
|
138
|
+
The credentials can also be provided through various environment variables depending on the
|
|
139
|
+
LangChain module being used. For the list of supported providers and the supported environment
|
|
140
|
+
variables credentials, please refer to the following table:
|
|
141
|
+
https://python.langchain.com/docs/integrations/chat/#featured-providers
|
|
142
|
+
|
|
143
|
+
# Using LiteLLM
|
|
144
|
+
```python
|
|
145
|
+
os.environ["OPENAI_API_KEY"] = "sk-..."
|
|
146
|
+
lm_invoker = build_lm_invoker(
|
|
147
|
+
model_id="litellm/openai/gpt-4o-mini",
|
|
148
|
+
)
|
|
149
|
+
```
|
|
150
|
+
For the list of supported providers, please refer to the following page:
|
|
151
|
+
https://docs.litellm.ai/docs/providers/
|
|
152
|
+
|
|
153
|
+
# Using Portkey
|
|
154
|
+
Portkey supports multiple authentication methods with strict precedence order.
|
|
155
|
+
Authentication methods are mutually exclusive and cannot be combined.
|
|
156
|
+
|
|
157
|
+
## Config ID Authentication (Highest Precedence)
|
|
158
|
+
```python
|
|
159
|
+
lm_invoker = build_lm_invoker(
|
|
160
|
+
model_id="portkey/any-model",
|
|
161
|
+
credentials="portkey-api-key",
|
|
162
|
+
config={"config": "pc-openai-4f6905"}
|
|
163
|
+
)
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
## Model Catalog Authentication (Combined Format)
|
|
167
|
+
```python
|
|
168
|
+
lm_invoker = build_lm_invoker(
|
|
169
|
+
model_id="portkey/@openai-custom/gpt-4o",
|
|
170
|
+
credentials="portkey-api-key"
|
|
171
|
+
)
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
## Model Catalog Authentication (Separate Parameters)
|
|
175
|
+
```python
|
|
176
|
+
lm_invoker = build_lm_invoker(
|
|
177
|
+
model_id="portkey/gpt-4o",
|
|
178
|
+
credentials="portkey-api-key",
|
|
179
|
+
config={"provider": "@openai-custom"}
|
|
180
|
+
)
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
## Direct Provider Authentication
|
|
184
|
+
```python
|
|
185
|
+
lm_invoker = build_lm_invoker(
|
|
186
|
+
model_id="portkey/gpt-4o",
|
|
187
|
+
credentials={
|
|
188
|
+
"portkey_api_key": "portkey-api-key",
|
|
189
|
+
"api_key": "sk-...", # Provider\'s API key
|
|
190
|
+
"provider": "openai" # Direct provider (no \'@\' prefix)
|
|
191
|
+
}
|
|
192
|
+
)
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
## Custom Host Override
|
|
196
|
+
```python
|
|
197
|
+
lm_invoker = build_lm_invoker(
|
|
198
|
+
model_id="portkey/@custom-provider/gpt-4o",
|
|
199
|
+
credentials="portkey-api-key",
|
|
200
|
+
config={"custom_host": "https://your-custom-endpoint.com"}
|
|
201
|
+
)
|
|
202
|
+
```
|
|
203
|
+
|
|
204
|
+
The Portkey API key can also be provided through the `PORTKEY_API_KEY` environment variable.
|
|
205
|
+
For more details on authentication methods, please refer to:
|
|
206
|
+
https://portkey.ai/docs/product/ai-gateway/universal-api
|
|
207
|
+
|
|
208
|
+
# Using xAI
|
|
209
|
+
```python
|
|
210
|
+
lm_invoker = build_lm_invoker(
|
|
211
|
+
model_id="xai/grok-3",
|
|
212
|
+
credentials="xai-..."
|
|
213
|
+
)
|
|
214
|
+
```
|
|
215
|
+
The credentials can also be provided through the `XAI_API_KEY` environment variable.
|
|
216
|
+
For the list of supported models, please refer to the following page:
|
|
217
|
+
https://docs.x.ai/docs/models
|
|
218
|
+
|
|
219
|
+
Security warning:
|
|
220
|
+
Please provide the LM invoker credentials ONLY to the `credentials` parameter. Do not put any kind of
|
|
221
|
+
credentials in the `config` parameter as the content of the `config` parameter will be logged.
|
|
222
|
+
'''
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
from _typeshed import Incomplete
|
|
2
|
+
from gllm_inference.builder.build_lm_invoker import build_lm_invoker as build_lm_invoker
|
|
3
|
+
from gllm_inference.builder.build_output_parser import build_output_parser as build_output_parser
|
|
4
|
+
from gllm_inference.prompt_builder import PromptBuilder as PromptBuilder
|
|
5
|
+
from gllm_inference.request_processor.lm_request_processor import LMRequestProcessor as LMRequestProcessor
|
|
6
|
+
from gllm_inference.schema.model_id import ModelId as ModelId
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
logger: Incomplete
|
|
10
|
+
|
|
11
|
+
def build_lm_request_processor(model_id: str | ModelId, credentials: str | dict[str, Any] | None = None, config: dict[str, Any] | None = None, system_template: str = '', user_template: str = '', key_defaults: dict[str, Any] | None = None, output_parser_type: str = 'none') -> LMRequestProcessor:
|
|
12
|
+
'''Build a language model invoker based on the provided configurations.
|
|
13
|
+
|
|
14
|
+
Args:
|
|
15
|
+
model_id (str | ModelId): The model id, can either be a ModelId instance or a string in a format defined
|
|
16
|
+
in the following page: https://gdplabs.gitbook.io/sdk/resources/supported-models#language-models-lms
|
|
17
|
+
credentials (str | dict[str, Any] | None, optional): The credentials for the language model. Can either be:
|
|
18
|
+
1. An API key.
|
|
19
|
+
2. A path to a credentials JSON file, currently only supported for Google Vertex AI.
|
|
20
|
+
3. A dictionary of credentials, currently supported for Bedrock and LangChain.
|
|
21
|
+
Defaults to None, in which case the credentials will be loaded from the appropriate environment variables.
|
|
22
|
+
config (dict[str, Any] | None, optional): Additional configuration for the language model. Defaults to None.
|
|
23
|
+
system_template (str): The system prompt template. May contain placeholders enclosed in curly braces `{}`.
|
|
24
|
+
Defaults to an empty string.
|
|
25
|
+
user_template (str): The user prompt template. May contain placeholders enclosed in curly braces `{}`.
|
|
26
|
+
Defaults to an empty string.
|
|
27
|
+
key_defaults (dict[str, str] | None, optional): Default values for the keys in the prompt templates.
|
|
28
|
+
Applied when the corresponding keys are not provided in the runtime input.
|
|
29
|
+
Defaults to None, in which case no default values will be assigned to the keys.
|
|
30
|
+
output_parser_type (str, optional): The type of output parser to use. Supports "json" and "none".
|
|
31
|
+
Defaults to "none".
|
|
32
|
+
|
|
33
|
+
Returns:
|
|
34
|
+
LMRequestProcessor: The initialized language model request processor.
|
|
35
|
+
|
|
36
|
+
Raises:
|
|
37
|
+
ValueError: If the provided configuration is invalid.
|
|
38
|
+
|
|
39
|
+
Usage examples:
|
|
40
|
+
```python
|
|
41
|
+
# Basic usage
|
|
42
|
+
lm_request_processor = build_lm_request_processor(
|
|
43
|
+
model_id="openai/gpt-4o-mini",
|
|
44
|
+
credentials="sk-...",
|
|
45
|
+
user_template="{query}",
|
|
46
|
+
)
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
# With custom LM invoker configuration
|
|
50
|
+
```python
|
|
51
|
+
config = {
|
|
52
|
+
"default_hyperparameters": {"temperature": 0.5},
|
|
53
|
+
"tools": [tool_1, tool_2],
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
lm_request_processor = build_lm_request_processor(
|
|
57
|
+
model_id="openai/gpt-4o-mini",
|
|
58
|
+
credentials="sk-...",
|
|
59
|
+
config=config,
|
|
60
|
+
user_template="{query}",
|
|
61
|
+
)
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
# With custom prompt builder configuration
|
|
65
|
+
```python
|
|
66
|
+
lm_request_processor = build_lm_request_processor(
|
|
67
|
+
model_id="openai/gpt-4o-mini",
|
|
68
|
+
credentials="sk-...",
|
|
69
|
+
system_template="Talk like a {role}.",
|
|
70
|
+
user_template="{query}",
|
|
71
|
+
key_defaults={"role": "pirate"},
|
|
72
|
+
)
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
# With output parser
|
|
76
|
+
```python
|
|
77
|
+
lm_request_processor = build_lm_request_processor(
|
|
78
|
+
model_id="openai/gpt-4o-mini",
|
|
79
|
+
credentials="sk-...",
|
|
80
|
+
user_template="{query}",
|
|
81
|
+
output_parser_type="json",
|
|
82
|
+
)
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
Security warning:
|
|
86
|
+
Please provide the LM invoker credentials ONLY to the `credentials` parameter. Do not put any kind of
|
|
87
|
+
credentials in the `config` parameter as the content of the `config` parameter will be logged.
|
|
88
|
+
'''
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
from _typeshed import Incomplete
|
|
2
|
+
from gllm_inference.output_parser import JSONOutputParser as JSONOutputParser
|
|
3
|
+
from gllm_inference.output_parser.output_parser import BaseOutputParser as BaseOutputParser
|
|
4
|
+
|
|
5
|
+
OUTPUT_PARSER_TYPE_MAP: Incomplete
|
|
6
|
+
|
|
7
|
+
def build_output_parser(output_parser_type: str) -> BaseOutputParser | None:
|
|
8
|
+
'''Build an output parser based on the provided configurations.
|
|
9
|
+
|
|
10
|
+
Args:
|
|
11
|
+
output_parser_type (str): The type of output parser to use. Supports "json" and "none".
|
|
12
|
+
|
|
13
|
+
Returns:
|
|
14
|
+
BaseOutputParser: The initialized output parser.
|
|
15
|
+
|
|
16
|
+
Raises:
|
|
17
|
+
ValueError: If the provided type is not supported.
|
|
18
|
+
|
|
19
|
+
Usage examples:
|
|
20
|
+
# Using JSON output parser
|
|
21
|
+
```python
|
|
22
|
+
output_parser = build_output_parser(output_parser_type="json")
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
# Not using output parser
|
|
26
|
+
```python
|
|
27
|
+
output_parser = build_output_parser(output_parser_type="none")
|
|
28
|
+
```
|
|
29
|
+
'''
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
from gllm_inference.catalog.lm_request_processor_catalog import LMRequestProcessorCatalog as LMRequestProcessorCatalog
|
|
2
|
+
from gllm_inference.catalog.prompt_builder_catalog import PromptBuilderCatalog as PromptBuilderCatalog
|
|
3
|
+
|
|
4
|
+
__all__ = ['LMRequestProcessorCatalog', 'PromptBuilderCatalog']
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import abc
|
|
2
|
+
from _typeshed import Incomplete
|
|
3
|
+
from abc import ABC
|
|
4
|
+
from pydantic import BaseModel
|
|
5
|
+
from typing import Generic, TypeVar
|
|
6
|
+
|
|
7
|
+
T = TypeVar('T')
|
|
8
|
+
logger: Incomplete
|
|
9
|
+
|
|
10
|
+
class BaseCatalog(ABC, BaseModel, Generic[T], arbitrary_types_allowed=True, metaclass=abc.ABCMeta):
|
|
11
|
+
'''A base class for catalogs used for loading and managing various components in GLLM Inference.
|
|
12
|
+
|
|
13
|
+
Attributes:
|
|
14
|
+
components (dict[str, T]): A dictionary containing the components.
|
|
15
|
+
|
|
16
|
+
Initialization:
|
|
17
|
+
# Example 1: Load from Google Sheets using client email and private key
|
|
18
|
+
```python
|
|
19
|
+
catalog = BaseCatalog.from_gsheets(
|
|
20
|
+
sheet_id="...",
|
|
21
|
+
worksheet_id="...",
|
|
22
|
+
client_email="...",
|
|
23
|
+
private_key="...",
|
|
24
|
+
)
|
|
25
|
+
component = catalog.name
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
# Example 2: Load from Google Sheets using credential file
|
|
29
|
+
```python
|
|
30
|
+
catalog = BaseCatalog.from_gsheets(
|
|
31
|
+
sheet_id="...",
|
|
32
|
+
worksheet_id="...",
|
|
33
|
+
credential_file_path="...",
|
|
34
|
+
)
|
|
35
|
+
component = catalog.name
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
# Example 3: Load from CSV
|
|
39
|
+
```python
|
|
40
|
+
catalog = BaseCatalog.from_csv(csv_path="...")
|
|
41
|
+
component = catalog.name
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
# Example 4: Load from records
|
|
45
|
+
```python
|
|
46
|
+
records = [
|
|
47
|
+
{"name": "...", "col_1": "...", "col_2": "..."},
|
|
48
|
+
{"name": "...", "col_1": "...", "col_2": "..."},
|
|
49
|
+
]
|
|
50
|
+
catalog = BaseCatalog.from_records(records=records)
|
|
51
|
+
component = catalog.name
|
|
52
|
+
```
|
|
53
|
+
'''
|
|
54
|
+
components: dict[str, T]
|
|
55
|
+
def __getattr__(self, name: str) -> T:
|
|
56
|
+
"""Fetches a component by attribute name.
|
|
57
|
+
|
|
58
|
+
This method attempts to retrieve a component from the `components` dictionary using the provided
|
|
59
|
+
attribute name. If the attribute is not found, it raises an `AttributeError`.
|
|
60
|
+
|
|
61
|
+
Args:
|
|
62
|
+
name (str): The name of the attribute to fetch.
|
|
63
|
+
|
|
64
|
+
Returns:
|
|
65
|
+
T: The component associated with the given attribute name.
|
|
66
|
+
|
|
67
|
+
Raises:
|
|
68
|
+
AttributeError: If the attribute name does not exist in the `components` dictionary.
|
|
69
|
+
"""
|
|
70
|
+
@classmethod
|
|
71
|
+
def from_gsheets(cls, sheet_id: str, worksheet_id: str = '0', credential_file_path: str = None, client_email: str = None, private_key: str = None) -> BaseCatalog[T]:
|
|
72
|
+
'''Creates a `BaseCatalog[T]` instance from Google Sheets data.
|
|
73
|
+
|
|
74
|
+
This class method reads component data from a Google Sheets worksheet and initializes components
|
|
75
|
+
based on the provided data. Authentication can be provided either by specifying the path to a `credential.json`
|
|
76
|
+
file or by directly supplying the `client_email` and `private_key`.
|
|
77
|
+
|
|
78
|
+
Args:
|
|
79
|
+
sheet_id (str): The ID of the Google Sheet.
|
|
80
|
+
worksheet_id (str): The ID of the worksheet within the Google Sheet. Defaults to "0"
|
|
81
|
+
credential_file_path (str): The file path to the `credential.json` file. If provided, `client_email` and
|
|
82
|
+
`private_key` are extracted from this file, effectively ignoring the `client_email` and `private_key`
|
|
83
|
+
arguments. Defaults to None.
|
|
84
|
+
client_email (str): The client email associated with the service account. This is ignored if
|
|
85
|
+
`credential_file_path` is provided. Defaults to None.
|
|
86
|
+
private_key (str): The private key used for authentication. This is ignored if `credential_file_path`
|
|
87
|
+
is provided. Defaults to None.
|
|
88
|
+
|
|
89
|
+
Returns:
|
|
90
|
+
BaseCatalog[T]: An instance of `BaseCatalog[T]` initialized with components based on the
|
|
91
|
+
Google Sheets data.
|
|
92
|
+
|
|
93
|
+
Raises:
|
|
94
|
+
ValueError: If authentication credentials are not provided or are invalid.
|
|
95
|
+
'''
|
|
96
|
+
@classmethod
|
|
97
|
+
def from_csv(cls, csv_path: str) -> BaseCatalog[T]:
|
|
98
|
+
"""Creates a `BaseCatalog[T]` instance from CSV data.
|
|
99
|
+
|
|
100
|
+
This class method reads component data from a CSV file and initializes components based on the provided data.
|
|
101
|
+
|
|
102
|
+
Args:
|
|
103
|
+
csv_path (str): The file path to the CSV containing component data.
|
|
104
|
+
|
|
105
|
+
Returns:
|
|
106
|
+
BaseCatalog[T]: An instance of `BaseCatalog[T]` initialized with components based on the
|
|
107
|
+
CSV data.
|
|
108
|
+
"""
|
|
109
|
+
@classmethod
|
|
110
|
+
def from_records(cls, records: list[dict[str, str]]) -> BaseCatalog[T]:
|
|
111
|
+
"""Creates a `BaseCatalog[T]` instance from a list of records.
|
|
112
|
+
|
|
113
|
+
This class method builds a catalog from the provided records.
|
|
114
|
+
|
|
115
|
+
Args:
|
|
116
|
+
records (list[dict[str, str]]): A list of records containing component data.
|
|
117
|
+
|
|
118
|
+
Returns:
|
|
119
|
+
BaseCatalog[T]: An instance of `BaseCatalog[T]` initialized with components based on the
|
|
120
|
+
list of records.
|
|
121
|
+
"""
|