ASUllmAPI 0.0.1__tar.gz

ASUllmAPI-0.0.1/ASUllmAPI/__init__.py

@@ -0,0 +1,2 @@
+from .model_config import ModelConfig
+from .api import query_llm, query_model_info_api, model_provider_mapper, model_list

ASUllmAPI-0.0.1/ASUllmAPI/api.py

@@ -0,0 +1,81 @@
+import requests
+import traceback
+from time import sleep
+from .model_config import ModelConfig
+
+__author__ = 'swliu'
+
+
+def query_llm(model: ModelConfig, query, num_retry: int = 3) -> str:
+    for i in range(num_retry):
+        try:
+            payload = _compute_payload(model=model, query=query)
+            headers = _compute_headers(model.access_token)
+            response = requests.post(model.api_url, json=payload, headers=headers)
+            response_text = response.json()['response']
+            return response_text
+        except Exception as e:
+            print(traceback.format_exc())
+            sleep(1)
+            continue
+    return ""
+
+
+def query_model_info_api(access_token, url, num_retry: int = 3) -> list:
+    """
+    :param access_token: API access token.
+    :param url: API endpoint
+    :param num_retry: int
+    :return: a list of large language models that are hosted by ASU
+    """
+    for i in range(num_retry):
+        try:
+            headers = _compute_headers(access_token=access_token)
+            response = requests.get(url=url, headers=headers)
+            models_dict = json.loads(response.content)
+            models = models_dict["models"]
+            return models
+        except Exception as e:
+            print(traceback.format_exc())
+            sleep(1)
+            continue
+    return []
+
+
+def model_provider_mapper(models: list) -> dict:
+    mapper = {
+        model["name"]: model["provider"]
+        for model in models
+    }
+    return mapper
+
+
+def model_list(models: list) -> set:
+    models = {model["name"] for model in models}
+    return models
+
+
+def _compute_headers(access_token):
+    headers = {
+        "Accept": "application/json",
+        "Authorization": f'Bearer {access_token}'
+    }
+    return headers
+
+
+def _compute_payload(model: ModelConfig, query: str):
+    payload = {
+        # required
+        "model_provider": model.provider,
+        # required
+        "model_name": model.name,
+        # optional
+        "model_params": model.model_params,
+        # required
+        "prompt": query,
+        # optional
+        "enable_search": model.enable_search,
+        # optional
+        "search_params": model.search_params
+    }
+    return payload

ASUllmAPI-0.0.1/ASUllmAPI/model_config.py

@@ -0,0 +1,68 @@
+from dataclasses import dataclass
+
+__author__ = 'swliu'
+
+
+@dataclass(frozen=True)
+class ModelConfig:
+    name: str = ''
+    provider: str = ''
+    enable_search: bool = True
+    api_url: str = ''
+    access_token: str = None
+    model_temperature: float = None
+    model_max_tokens: int = None
+    model_top_p = None
+    model_top_k = None
+    search_collection: str = 'asu'
+    search_top_k: int = 3
+
+    @property
+    def model_params(self):
+        if not self.access_token:
+            raise AccessTokenMissing
+        if not self.api_url:
+            raise APIUrlMissing
+
+        model_params = {}
+        if self.model_temperature is not None:
+            model_params["temperature"] = self.model_temperature
+        if self.model_max_tokens is not None:
+            model_params["max_tokens"] = self.model_max_tokens
+        if self.model_top_p is not None:
+            model_params["top_p"] = self.model_top_p
+        if self.model_top_k is not None:
+            model_params["top_k"] = self.model_top_k
+        return model_params
+
+    @property
+    def search_params(self):
+        if not self.access_token:
+            raise AccessTokenMissing
+        if not self.api_url:
+            raise APIUrlMissing
+
+        search_params = {}
+        if self.enable_search is True:
+            search_params["collection"] = self.search_collection
+            search_params["top_k"] = self.search_top_k
+        return search_params
+
+    def __str__(self):
+        if self.enable_search:
+            return f"{self.name}_search_enabled"
+        else:
+            return self.name
+
+    def __repr__(self):
+        return f"Model: {self.name}\tSearch Enabled: {str(self.enable_search)}"
+
+
+class AccessTokenMissing(Exception):
+    def __init__(self):
+        self.message = "API access token is missing."
+
+
+class APIUrlMissing(Exception):
+    def __init__(self):
+        self.message = "API url token is missing."

ASUllmAPI-0.0.1/ASUllmAPI.egg-info/PKG-INFO

@@ -0,0 +1,136 @@
+Metadata-Version: 2.1
+Name: ASUllmAPI
+Version: 0.0.1
+Summary: A simple python package to facilitate connection to ASU LLM API
+Author-email: Stella Wenxing Liu <stellawenxingliu@gmail.com>
+Project-URL: Homepage, https://github.com/ASU/aiml-ssmdv-student-support-ml-data-visualization
+Project-URL: Issues, https://github.com/ASU/aiml-ssmdv-student-support-ml-data-visualization/issues
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+
+# Models
+
+### Model Params
+```markdown
+"model_params": {
+    "temperature": float,
+    "max_tokens": int,
+    "top_p": float,
+    "top_k": int,
+},
+"enable_search": bool,
+"search_params": {
+    "collection": "asu",
+    "top_k": int,
+}
+```
+#### temperature
+Randomness and Diversity parameter. Use a lower value to decrease randomness in the response.
+#### top_p
+Randomness and Diversity parameter. Use a lower value to ignore less probable options.
+#### top_k
+Randomness and Diversity parameter. The number of token choices the model uses to generate the next token.
+#### max_tokens
+The maximum number of tokens in the generated response.
+
+## Provider: AES
+Checkout [Amazon Bedrock User Guide](https://docs.aws.amazon.com/bedrock/latest/userguide/model-ids-arns.html) for base model ids and versions.
+
+### Titan Text Models
+
+Model IDs:
+- titang1lite
+- titang1express
+
+[Model documentation](https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-titan-text.html):
+
+| Category                 | Parameter   | Key         | Minimum | Maximum | Default |
+|--------------------------|-------------|-------------|---------|---------|---------|
+| Randomness and diversity | Temperature | temperature | 0       | 1       | 0.5     |
+| Randomness and diversity | Top P       | top_p       | 0       | 1       | 1       |
+| Length                   | max_tokens  | max_tokens  | 0       | 8000    | 512     |
+
+
+### Anthropic Claude models
+
+Model IDs:
+- claude2_1 
+- claude2 
+- claude1_3 
+- claudeinstant
+
+[Model documentation](https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-claude.html):
+
+| Category                 | Parameter   | Key         | Minimum | Maximum | Default |
+|--------------------------|-------------|-------------|---------|---------|---------|
+| Randomness and diversity | Temperature | temperature | 0       | 1       | 0.5     |
+| Randomness and diversity | Top P       | top_p       | 0       | 1       | 0.5     |
+| Randomness and diversity | Top K       | top_k       | 0       | 500     | 250     |
+| Length                   | max_tokens  | max_tokens  | 0       | 4096    | 200     |
+
+### AI21 Labs Jurassic-2 models
+
+Model IDs:
+- j2ultra
+- j2mid
+
+[Model documentation](https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-jurassic2.html)
+
+| Category                 | Parameter   | Key         | Minimum | Maximum | Default |
+|--------------------------|-------------|-------------|---------|---------|---------|
+| Randomness and diversity | Temperature | temperature | 0       | 1       | 0.5     |
+| Randomness and diversity | Top P       | top_p       | 0       | 1       | 0.5     |
+| Length                   | max_tokens  | max_tokens  | 0       | 8191    | 200     |
+
+### Cohere Command Models
+
+Model IDs:
+- command
+- commandlight
+
+[Model documentation](https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-cohere-command.html)
+
+| Category                 | Parameter   | Key         | Minimum | Maximum | Default |
+|--------------------------|-------------|-------------|---------|---------|---------|
+| Randomness and diversity | Temperature | temperature | 0       | 5       | 0.9     |
+| Randomness and diversity | Top P       | top_p       | 0       | 1       | 0.75    |
+| Randomness and diversity | Top K       | top_k       | 0       | 500     | 0       |
+| Length                   | max_tokens  | max_tokens  | 1       | 4096    | 20      |
+
+### Meta Llama2 Models
+
+Model IDs:
+- llama2-13b
+
+[Model Documentation](https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-meta.html)
+
+| Category                 | Parameter   | Key         | Minimum | Maximum | Default |
+|--------------------------|-------------|-------------|---------|---------|---------|
+| Randomness and diversity | Temperature | temperature | 0       | 1       | 0.5     |
+| Randomness and diversity | Top P       | top_p       | 0       | 1       | 0.9     |
+| Length                   | max_tokens  | max_tokens  | 1       | 2048    | 512     |
+
+
+## Provider: Azure
+
+[Model Documentation](https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models#model-summary-table-and-region-availability)
+
+| Model name | Model ID         | Max Request (tokens) |
+|------------|------------------|----------------------|
+| gpt3_5     | gpt-35-turbo     | 4096                 |
+| gpt3_5-16k | gpt-35-turbo-16k | 16384                |
+| gpt4       | gpt-4            | 8192                 |
+| gpt4-32k   | gpt-4-32k        | 32768                |
+
+## Provider: GCP
+
+[Model Documentation](https://cloud.google.com/vertex-ai/docs/generative-ai/learn/models)
+
+| Model name          | Model ID       | Max Input Tokens     | Max Output Tokens |
+|---------------------|----------------|----------------------|-------------------|
+| PaLM 2 for Chat     | chat-bison     | 8192                 | 1024              |
+| PaLM 2 for Chat 32k | chat-bison-32k | 32768 (input+output) | 8192              |

ASUllmAPI-0.0.1/ASUllmAPI.egg-info/SOURCES.txt

@@ -0,0 +1,10 @@
+LICENSE
+README.md
+pyproject.toml
+ASUllmAPI/__init__.py
+ASUllmAPI/api.py
+ASUllmAPI/model_config.py
+ASUllmAPI.egg-info/PKG-INFO
+ASUllmAPI.egg-info/SOURCES.txt
+ASUllmAPI.egg-info/dependency_links.txt
+ASUllmAPI.egg-info/top_level.txt

ASUllmAPI-0.0.1/ASUllmAPI.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

ASUllmAPI-0.0.1/ASUllmAPI.egg-info/top_level.txt

@@ -0,0 +1 @@
+ASUllmAPI

ASUllmAPI-0.0.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 AI Acceleration Team at Arizona State University
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

ASUllmAPI-0.0.1/PKG-INFO

@@ -0,0 +1,136 @@
+Metadata-Version: 2.1
+Name: ASUllmAPI
+Version: 0.0.1
+Summary: A simple python package to facilitate connection to ASU LLM API
+Author-email: Stella Wenxing Liu <stellawenxingliu@gmail.com>
+Project-URL: Homepage, https://github.com/ASU/aiml-ssmdv-student-support-ml-data-visualization
+Project-URL: Issues, https://github.com/ASU/aiml-ssmdv-student-support-ml-data-visualization/issues
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+
+# Models
+
+### Model Params
+```markdown
+"model_params": {
+    "temperature": float,
+    "max_tokens": int,
+    "top_p": float,
+    "top_k": int,
+},
+"enable_search": bool,
+"search_params": {
+    "collection": "asu",
+    "top_k": int,
+}
+```
+#### temperature
+Randomness and Diversity parameter. Use a lower value to decrease randomness in the response.
+#### top_p
+Randomness and Diversity parameter. Use a lower value to ignore less probable options.
+#### top_k
+Randomness and Diversity parameter. The number of token choices the model uses to generate the next token.
+#### max_tokens
+The maximum number of tokens in the generated response.
+
+## Provider: AES
+Checkout [Amazon Bedrock User Guide](https://docs.aws.amazon.com/bedrock/latest/userguide/model-ids-arns.html) for base model ids and versions.
+
+### Titan Text Models
+
+Model IDs:
+- titang1lite
+- titang1express
+
+[Model documentation](https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-titan-text.html):
+
+| Category                 | Parameter   | Key         | Minimum | Maximum | Default |
+|--------------------------|-------------|-------------|---------|---------|---------|
+| Randomness and diversity | Temperature | temperature | 0       | 1       | 0.5     |
+| Randomness and diversity | Top P       | top_p       | 0       | 1       | 1       |
+| Length                   | max_tokens  | max_tokens  | 0       | 8000    | 512     |
+
+
+### Anthropic Claude models
+
+Model IDs:
+- claude2_1 
+- claude2 
+- claude1_3 
+- claudeinstant
+
+[Model documentation](https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-claude.html):
+
+| Category                 | Parameter   | Key         | Minimum | Maximum | Default |
+|--------------------------|-------------|-------------|---------|---------|---------|
+| Randomness and diversity | Temperature | temperature | 0       | 1       | 0.5     |
+| Randomness and diversity | Top P       | top_p       | 0       | 1       | 0.5     |
+| Randomness and diversity | Top K       | top_k       | 0       | 500     | 250     |
+| Length                   | max_tokens  | max_tokens  | 0       | 4096    | 200     |
+
+### AI21 Labs Jurassic-2 models
+
+Model IDs:
+- j2ultra
+- j2mid
+
+[Model documentation](https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-jurassic2.html)
+
+| Category                 | Parameter   | Key         | Minimum | Maximum | Default |
+|--------------------------|-------------|-------------|---------|---------|---------|
+| Randomness and diversity | Temperature | temperature | 0       | 1       | 0.5     |
+| Randomness and diversity | Top P       | top_p       | 0       | 1       | 0.5     |
+| Length                   | max_tokens  | max_tokens  | 0       | 8191    | 200     |
+
+### Cohere Command Models
+
+Model IDs:
+- command
+- commandlight
+
+[Model documentation](https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-cohere-command.html)
+
+| Category                 | Parameter   | Key         | Minimum | Maximum | Default |
+|--------------------------|-------------|-------------|---------|---------|---------|
+| Randomness and diversity | Temperature | temperature | 0       | 5       | 0.9     |
+| Randomness and diversity | Top P       | top_p       | 0       | 1       | 0.75    |
+| Randomness and diversity | Top K       | top_k       | 0       | 500     | 0       |
+| Length                   | max_tokens  | max_tokens  | 1       | 4096    | 20      |
+
+### Meta Llama2 Models
+
+Model IDs:
+- llama2-13b
+
+[Model Documentation](https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-meta.html)
+
+| Category                 | Parameter   | Key         | Minimum | Maximum | Default |
+|--------------------------|-------------|-------------|---------|---------|---------|
+| Randomness and diversity | Temperature | temperature | 0       | 1       | 0.5     |
+| Randomness and diversity | Top P       | top_p       | 0       | 1       | 0.9     |
+| Length                   | max_tokens  | max_tokens  | 1       | 2048    | 512     |
+
+
+## Provider: Azure
+
+[Model Documentation](https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models#model-summary-table-and-region-availability)
+
+| Model name | Model ID         | Max Request (tokens) |
+|------------|------------------|----------------------|
+| gpt3_5     | gpt-35-turbo     | 4096                 |
+| gpt3_5-16k | gpt-35-turbo-16k | 16384                |
+| gpt4       | gpt-4            | 8192                 |
+| gpt4-32k   | gpt-4-32k        | 32768                |
+
+## Provider: GCP
+
+[Model Documentation](https://cloud.google.com/vertex-ai/docs/generative-ai/learn/models)
+
+| Model name          | Model ID       | Max Input Tokens     | Max Output Tokens |
+|---------------------|----------------|----------------------|-------------------|
+| PaLM 2 for Chat     | chat-bison     | 8192                 | 1024              |
+| PaLM 2 for Chat 32k | chat-bison-32k | 32768 (input+output) | 8192              |

ASUllmAPI-0.0.1/README.md

@@ -0,0 +1,122 @@
+# Models
+
+### Model Params
+```markdown
+"model_params": {
+    "temperature": float,
+    "max_tokens": int,
+    "top_p": float,
+    "top_k": int,
+},
+"enable_search": bool,
+"search_params": {
+    "collection": "asu",
+    "top_k": int,
+}
+```
+#### temperature
+Randomness and Diversity parameter. Use a lower value to decrease randomness in the response.
+#### top_p
+Randomness and Diversity parameter. Use a lower value to ignore less probable options.
+#### top_k
+Randomness and Diversity parameter. The number of token choices the model uses to generate the next token.
+#### max_tokens
+The maximum number of tokens in the generated response.
+
+## Provider: AES
+Checkout [Amazon Bedrock User Guide](https://docs.aws.amazon.com/bedrock/latest/userguide/model-ids-arns.html) for base model ids and versions.
+
+### Titan Text Models
+
+Model IDs:
+- titang1lite
+- titang1express
+
+[Model documentation](https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-titan-text.html):
+
+| Category                 | Parameter   | Key         | Minimum | Maximum | Default |
+|--------------------------|-------------|-------------|---------|---------|---------|
+| Randomness and diversity | Temperature | temperature | 0       | 1       | 0.5     |
+| Randomness and diversity | Top P       | top_p       | 0       | 1       | 1       |
+| Length                   | max_tokens  | max_tokens  | 0       | 8000    | 512     |
+
+
+### Anthropic Claude models
+
+Model IDs:
+- claude2_1 
+- claude2 
+- claude1_3 
+- claudeinstant
+
+[Model documentation](https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-claude.html):
+
+| Category                 | Parameter   | Key         | Minimum | Maximum | Default |
+|--------------------------|-------------|-------------|---------|---------|---------|
+| Randomness and diversity | Temperature | temperature | 0       | 1       | 0.5     |
+| Randomness and diversity | Top P       | top_p       | 0       | 1       | 0.5     |
+| Randomness and diversity | Top K       | top_k       | 0       | 500     | 250     |
+| Length                   | max_tokens  | max_tokens  | 0       | 4096    | 200     |
+
+### AI21 Labs Jurassic-2 models
+
+Model IDs:
+- j2ultra
+- j2mid
+
+[Model documentation](https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-jurassic2.html)
+
+| Category                 | Parameter   | Key         | Minimum | Maximum | Default |
+|--------------------------|-------------|-------------|---------|---------|---------|
+| Randomness and diversity | Temperature | temperature | 0       | 1       | 0.5     |
+| Randomness and diversity | Top P       | top_p       | 0       | 1       | 0.5     |
+| Length                   | max_tokens  | max_tokens  | 0       | 8191    | 200     |
+
+### Cohere Command Models
+
+Model IDs:
+- command
+- commandlight
+
+[Model documentation](https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-cohere-command.html)
+
+| Category                 | Parameter   | Key         | Minimum | Maximum | Default |
+|--------------------------|-------------|-------------|---------|---------|---------|
+| Randomness and diversity | Temperature | temperature | 0       | 5       | 0.9     |
+| Randomness and diversity | Top P       | top_p       | 0       | 1       | 0.75    |
+| Randomness and diversity | Top K       | top_k       | 0       | 500     | 0       |
+| Length                   | max_tokens  | max_tokens  | 1       | 4096    | 20      |
+
+### Meta Llama2 Models
+
+Model IDs:
+- llama2-13b
+
+[Model Documentation](https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-meta.html)
+
+| Category                 | Parameter   | Key         | Minimum | Maximum | Default |
+|--------------------------|-------------|-------------|---------|---------|---------|
+| Randomness and diversity | Temperature | temperature | 0       | 1       | 0.5     |
+| Randomness and diversity | Top P       | top_p       | 0       | 1       | 0.9     |
+| Length                   | max_tokens  | max_tokens  | 1       | 2048    | 512     |
+
+
+## Provider: Azure
+
+[Model Documentation](https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models#model-summary-table-and-region-availability)
+
+| Model name | Model ID         | Max Request (tokens) |
+|------------|------------------|----------------------|
+| gpt3_5     | gpt-35-turbo     | 4096                 |
+| gpt3_5-16k | gpt-35-turbo-16k | 16384                |
+| gpt4       | gpt-4            | 8192                 |
+| gpt4-32k   | gpt-4-32k        | 32768                |
+
+## Provider: GCP
+
+[Model Documentation](https://cloud.google.com/vertex-ai/docs/generative-ai/learn/models)
+
+| Model name          | Model ID       | Max Input Tokens     | Max Output Tokens |
+|---------------------|----------------|----------------------|-------------------|
+| PaLM 2 for Chat     | chat-bison     | 8192                 | 1024              |
+| PaLM 2 for Chat 32k | chat-bison-32k | 32768 (input+output) | 8192              |

ASUllmAPI-0.0.1/pyproject.toml

@@ -0,0 +1,18 @@
+[project]
+name = "ASUllmAPI"
+version = "0.0.1"
+authors = [
+  { name="Stella Wenxing Liu", email="stellawenxingliu@gmail.com" },
+]
+description = "A simple python package to facilitate connection to ASU LLM API"
+readme = "README.md"
+requires-python = ">=3.8"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+]
+
+[project.urls]
+Homepage = "https://github.com/ASU/aiml-ssmdv-student-support-ml-data-visualization"
+Issues = "https://github.com/ASU/aiml-ssmdv-student-support-ml-data-visualization/issues"

ASUllmAPI-0.0.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+