vijil 0.1.0__tar.gz

vijil-0.1.0/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

vijil-0.1.0/PKG-INFO

@@ -0,0 +1,53 @@
+Metadata-Version: 2.1
+Name: vijil
+Version: 0.1.0
+Summary: Python Client for Vijil
+Author: Subho Majumdar
+Author-email: subho@vijil.ai
+Requires-Python: >=3.11,<4.0
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: fastparquet (>=2024.5.0,<2025.0.0)
+Requires-Dist: fpdf2 (>=2.7.9,<3.0.0)
+Requires-Dist: jinja2 (>=3.1.4,<4.0.0)
+Requires-Dist: pandas (>=2.2.2,<3.0.0)
+Requires-Dist: python-dotenv (>=1.0.1,<2.0.0)
+Requires-Dist: requests (>=2.32.2,<3.0.0)
+Description-Content-Type: text/markdown
+
+# vijil-python
+Python Client for Vijil
+
+
+## Setup
+
+```bash
+pip install git+https://github.com/vijilAI/vijil-python
+```
+
+Then initialize the client using
+
+```python
+from vijil import Vijil
+
+client = Vijil()
+```
+
+
+Requires a `VIJIL_API_KEY`, either loaded in the environment or suppllied as `api_key` argument above.
+
+## Run Evaluations
+
+```python
+client.evaluations.create(
+    model_hub="openai",
+    model_name="gpt-3.5-turbo",
+    model_params={"temperature": 0},
+    harnesses=["ethics","hallucination"],
+    harness_params={"sample_size": 5}
+)
+```
+
+See the [minimal example](tutorials/minimal_example.ipynb) for more functionalities.
+

vijil-0.1.0/README.md

@@ -0,0 +1,34 @@
+# vijil-python
+Python Client for Vijil
+
+
+## Setup
+
+```bash
+pip install git+https://github.com/vijilAI/vijil-python
+```
+
+Then initialize the client using
+
+```python
+from vijil import Vijil
+
+client = Vijil()
+```
+
+
+Requires a `VIJIL_API_KEY`, either loaded in the environment or suppllied as `api_key` argument above.
+
+## Run Evaluations
+
+```python
+client.evaluations.create(
+    model_hub="openai",
+    model_name="gpt-3.5-turbo",
+    model_params={"temperature": 0},
+    harnesses=["ethics","hallucination"],
+    harness_params={"sample_size": 5}
+)
+```
+
+See the [minimal example](tutorials/minimal_example.ipynb) for more functionalities.

vijil-0.1.0/pyproject.toml

@@ -0,0 +1,28 @@
+[tool.poetry]
+name = "vijil"
+version = "0.1.0"
+description = "Python Client for Vijil"
+authors = [
+    "Subho Majumdar <subho@vijil.ai>",
+    "Leif Hancox-Li <leif@vijil.ai>",
+    "Harsh Raj <harsh@vijil.ai>",
+    "Aditya Karan <aditya@vijil.ai>"
+]
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.11"
+requests = "^2.32.2"
+python-dotenv = "^1.0.1"
+pandas = "^2.2.2"
+jinja2 = "^3.1.4"
+fpdf2 = "^2.7.9"
+fastparquet = "^2024.5.0"
+
+
+[tool.poetry.group.dev.dependencies]
+ipykernel = "^6.29.4"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

vijil-0.1.0/vijil/__init__.py

@@ -0,0 +1 @@
+from ._client import Vijil

vijil-0.1.0/vijil/_client.py

@@ -0,0 +1,724 @@
+import os
+import uuid
+import pandas as pd
+import json
+import numpy as np
+from typing import List
+from datetime import datetime, timezone
+import jinja2
+from fpdf import FPDF
+
+from vijil.api import make_api_request, get_api_proxy_dict, BASE_URL, SUPPORTED_HUBS
+from vijil.types import VijilClient
+
+    
+class APIKeys:
+    """
+    Class for managing model hub API keys, which are required to query models.
+
+    :param client: The Vijil client instance.
+    :type client: VijilClient
+    """
+    
+    def __init__(self, client: VijilClient) -> None:
+        """Constructor class
+
+        :param client: The Vijil client instance.
+        :type client: VijilClient
+        """
+        self.endpoint = "api-keys"
+        self.client = client
+        pass
+    
+    def list(self):
+        """List all stored model hub API keys.
+
+        :return: List of dictionaries. Each dictionary contains information about an API key.
+        :rtype: List(dict)
+        """
+        response = make_api_request(base_url=self.client.base_url, endpoint=self.endpoint, token=self.client.api_key)
+        return response
+    
+    def get_id_by_name(self, name: str):
+        """ Get the id of a key by its name. Used by other functions to get the id of a key to modify.
+
+        :param name: The name of the key.
+        :type name: str
+        :return: The id of the key.
+        :rtype: str
+        """
+
+        if not self.name_exists(name):
+            raise ValueError(f"Key '{name}' does not exist. Please specify an existing name.")
+
+        #get id of key to modify
+        response = self.list()
+        return [item['id'] for item in response if item['name'] == name][0]
+    
+    def check_model_hub(self, model_hub: str):
+        """Used by other functions to check that the model hub is valid and the key name is unique
+
+        :param model_hub: The name of the model hub.
+        :type model_hub: str
+        :raises ValueError: If the model hub is not supported.
+        """
+
+        # check if model_hub is valid
+        if (model_hub is not None) and (model_hub not in SUPPORTED_HUBS.keys()):
+            raise ValueError(f"Model hub {model_hub} is not supported.")
+    
+    def name_exists(self, name: str):
+        """Check whether the API key name already exists. Some functions need it to be exist, others need it to not exist.
+        
+        :param name: The name of the key.
+        :type name: str
+        :return: True if the name exists among the stored API keys, False otherwise.
+        :rtype: bool
+        """
+        
+        response = self.list()
+        if name in [item['name'] for item in response]:
+            return True
+        else:
+            return False
+
+    def create(self, name: str, model_hub: str, api_key: str, rate_limit_per_interval=60, rate_limit_interval=10):
+        """Create a new model hub API key.
+
+        :param name: Name for the API key. This must be unique.
+        :type name: str
+        :param model_hub: Name of the model hub. Current supported values are 'openai', 'together', 'octo'.
+        :type model_hub: str
+        :param api_key: The API key.
+        :type api_key: str
+        :param rate_limit_per_interval: The maximum amount of times Vijil will query the model hub in the specified rate_limit_interval, defaults to 60
+        :type rate_limit_per_interval: int, optional
+        :param rate_limit_interval: The size of the interval (in seconds) defining maximum queries to model hub in said interval. For example, if rate_limit_per_interval is 60 and rate_limit_interval is 10, then Vijil will query the model hub at most 60 times in 10 seconds. Defaults to 10
+        :type rate_limit_interval: int, optional
+        :raises ValueError: If you try to create a key with a name that belongs to an existing key.
+        :return: Response to the API request.
+        :rtype: dict
+        """
+        #check that model hub is supported                
+        self.check_model_hub(model_hub=model_hub)
+
+        if self.name_exists(name):
+            raise ValueError(f"Key name '{name}' already exists. Please specify a different name.")
+
+        payload = {
+            "name": name,
+            "hub": model_hub,
+            "rate_limit_per_interval": rate_limit_per_interval,
+            "rate_limit_interval": rate_limit_interval,
+            "value": api_key,
+        }
+        
+        return make_api_request(
+            base_url=self.client.base_url,
+            endpoint=self.endpoint,
+            method="post",
+            data=payload,
+            token=self.client.api_key
+        )
+    
+    def rename(self, name: str, new_name: str):
+        """Rename a stored API key
+        
+        :param name: The current name of the key.
+        :type name: str
+        :param new_name: The new name of the key.
+        :type new_name: str
+        """
+
+        key_id = self.get_id_by_name(name=name)
+        
+        payload = {
+            "name": new_name
+        }
+
+        return make_api_request(
+            base_url=self.client.base_url,
+            endpoint=self.endpoint + "/" + key_id,
+            method="put",
+            data=payload,
+            token=self.client.api_key
+        )
+
+    
+    def modify(self, name: str, model_hub: str=None, api_key:str =None, rate_limit_per_interval=None, rate_limit_interval=None):
+        """Modify model hub, key, or rate limits of a stored API key. Cannot be used to rename key.
+        
+        :param name: The name of the key you want to modify.
+        :type name: str
+        :param model_hub: Name of the model hub. Current supported values are 'openai', 'together', 'octo'.
+        :type model_hub: str, optional
+        :param api_key: The API key.
+        :type api_key: str, optional
+        :param rate_limit_per_interval: The maximum amount of times Vijil will query the model hub in the specified rate_limit_interval, defaults to 60
+        :type rate_limit_per_interval: int, optional
+        :param rate_limit_interval: The size of the interval (in seconds) defining maximum queries to model hub in said interval. For example, if rate_limit_per_interval is 60 and rate_limit_interval is 10, then Vijil will query the model hub at most 60 times in 10 seconds. Defaults to 10
+        :type rate_limit_interval: int, optional
+        """
+
+        #check that model hub is supported and key name refers to existing name
+        if model_hub is not None:
+           self.check_model_hub(model_hub=model_hub)        
+        
+        key_id = self.get_id_by_name(name=name)
+
+        payload = {
+            "name": name,
+            "hub": model_hub,
+            "rate_limit_per_interval": rate_limit_per_interval,
+            "rate_limit_interval": rate_limit_interval,
+            "value": api_key,
+        }
+
+        #remove None values
+        payload = {k: v for k, v in payload.items() if v is not None}
+
+        return make_api_request(
+            base_url=self.client.base_url,
+            endpoint=self.endpoint + "/" + key_id,
+            method="put",
+            data=payload,
+            token=self.client.api_key
+        )
+
+    def delete(self, name: str):
+        """Delete API key with specified name
+        
+        :param name: The name of the key you want to delete
+        :type name: str
+        :return: Response to the API request.
+        :rtype: dict
+        """
+
+        key_id = self.get_id_by_name(name=name)
+
+        return make_api_request(
+            base_url=self.client.base_url,
+            endpoint=self.endpoint + "/" + key_id,
+            method="delete",
+            token=self.client.api_key
+        )
+
+
+class Harnesses:
+    """Class for handling harnesses API requests.
+    
+    :param client: The VijilClient instance.
+    :type client: VijilClient
+    """
+    
+    def __init__(self, client: VijilClient) -> None:
+        """Initialize the Harnesses class.
+
+        :param VijilClient client: The VijilClient instance.
+        :type client: VijilClient
+        """
+        self.endpoint = "harness-configs"
+        self.client = client
+        pass
+
+    def list(self):
+        """List all harnesses.
+
+        :return: List of dicts where each dict contains the metadata for a harness.
+        :rtype: List(dict)
+        """
+        response = make_api_request(base_url=self.client.base_url, endpoint=self.endpoint, token=self.client.api_key)
+        return response["results"]
+    
+    def create(self, harness_name: str, component_list: List[str], component_type: str = "probe"):
+        """Not implemented yet.
+        """
+        raise NotImplementedError("Create harnesses is not implemented yet.")
+
+        if component_type not in ["probe", "scenario"]:
+            raise ValueError("component_type must be either 'probe' or 'scenario'")
+        payload = {
+            "id": str(uuid.uuid4()),
+            "harness_name": harness_name,
+            "component_list": component_list,
+            "component_type": component_type
+        }
+        
+        #if request is succeessful, return the harness id, name, and a "submitted" message
+        return make_api_request(
+            base_url=self.client.base_url,
+            endpoint=self.endpoint,
+            method="post",
+            data=payload,
+            token=self.client.api_key
+        )
+    def delete(self, harness_id: str):
+        """
+        Not implemented yet.
+        """
+        raise NotImplementedError("Delete harnesses is not implemented yet.")
+        
+        #if request is succeessful, return the harness id, name, and a "submitted" message
+        return make_api_request(
+            base_url=self.client.base_url,
+            endpoint=f"harness-configs/{harness_id}",
+            method="delete",
+            token=self.client.api_key
+        )
+
+
+    
+class Evaluations:
+    """Class for handling evaluations API requests.
+    
+    :param client: The VijilClient instance.
+    :type client: VijilClient
+    """
+    
+    def __init__(
+        self,
+        client: VijilClient
+    ) -> None:
+        self.endpoint = "evaluations"
+        self.client = client
+        self.api_proxy_dict = None
+    
+    def list(self, limit=10):
+        """List all valuations. Will return only 10 evaluations unless specified.
+        
+        :param limit: The number of evaluations to return, defaults to 10.
+        :type limit: int, optional
+        """
+
+        response = make_api_request(base_url=self.client.base_url, endpoint=self.endpoint+"?limit="+str(limit), token=self.client.api_key)
+        return response['results']
+    
+    def create(self, api_key_name: str=None, model_hub: str="openai", model_name: str="gpt-3.5-turbo", model_params={}, harness_params={}, harnesses=[]):
+        """Create a new evaluation.
+
+        :param api_key_name: The name of the model hub API key you want to use. If not specified, will use the first key we find for the specified model_hub.
+        :type api_key_name: str, optional
+        :param model_hub: The model hub you want to use. Supported options are "openai", "together", "octo".  Defaults to "openai"
+        :type model_hub: str, optional
+        :param model_name: The name of the model you want to use. Check the model hub's API documentation to find valid names. Defaults to "gpt-3.5-turbo"
+        :type model_name: str, optional
+        :param model_params: A dictionary specifying inference parameters like temperature and top_p. If none are specified, model hub defaults will be used. Defaults to {}
+        :type model_params: dict, optional
+        :param harness_params: Set optional parameters like is_lite, defaults to {}
+        :type harness_params: dict, optional
+        :param harnesses: A list of harnesses you want to include in the evaluation, defaults to []
+        :type harnesses: List[str], optional
+        :raises TypeError: If you have no API keys stored.
+        :raises ValueError: If you have no API keys stored for the specified model hub.
+        :raises ValueError: If you supply an api_key_name that does not exist.
+        :raises ValueError: If you specify lite mode for any harness other than ethics.
+        :return: API response containing evaluation ID of the newly created evaluation.
+        :rtype: dict
+        """
+
+
+        # store all api keys in a dictionary if not there
+        if self.api_proxy_dict is None:
+            try:
+                self.api_proxy_dict = get_api_proxy_dict(base_url=self.client.base_url, token=self.client.api_key)
+            except TypeError:
+                raise TypeError("No API keys found. Please upload an API key first.")
+            
+        # get api key proxy for the model hub
+        try:
+            hub_api_proxy_dict = self.api_proxy_dict[model_hub]
+        except KeyError:
+            raise ValueError(f"No API key stored for model hub {model_hub}")
+        
+        # get api key proxy for the model hub
+        if api_key_name is not None:
+            #find api key id for the api key name
+            if api_key_name not in hub_api_proxy_dict.keys():
+                raise ValueError(f"No API key found for name {api_key_name}")
+            else:
+                api_key_proxy = hub_api_proxy_dict[api_key_name]
+        else: #if no key specified, use first value in hub dictionary
+            api_key_proxy = next(iter(hub_api_proxy_dict.values()))
+
+        # create the payload
+        payload = { 
+                "id": str(uuid.uuid4()),
+                "hub": model_hub,
+                "model": model_name,
+                "api_key_proxy": api_key_proxy,
+                "scenario_config_filters": []
+        }
+        if model_params:
+            payload['agent_params'] = model_params
+            if model_params.get("num_generations"):
+                payload["num_generations"] = model_params["num_generations"]
+        if harness_params.get('sample_size'):
+            payload['sample_size'] = harness_params['sample_size']
+        if harness_params.get('is_lite')==True:
+            if harnesses == ['ethics']:
+                harnesses = ['ethics_lite']
+            else:
+                raise ValueError("Lite mode is only available for the ethics harness.")
+        if "trust_score" in harnesses:
+            if len(harnesses)==1:
+                harnesses_modified = ["security","privacy","hallucination","robustness","toxicity","stereotype","fairness","ethics","OpenLLM"]
+            else:
+                raise ValueError("The `trust_score` harness is available only as a standalone harness.")
+        else:
+            harnesses_modified = harnesses
+        payload["harness_config_ids"] = [f"vijil.harnesses.{h}" for h in harnesses_modified]
+        
+
+        return make_api_request(
+            base_url=self.client.base_url,
+            endpoint=self.endpoint,
+            method="post",
+            data=payload,
+            token=self.client.api_key
+        )
+        
+    def get_status(self, evaluation_id: str):
+        """Retrieve the status of an evaluation
+
+        :param evaluation_id: The unique ID of the evaluation
+        :type evaluation_id: str
+        :return: A dict with the id, status, and other metadata of the evaluation
+        :rtype: dict
+        """
+
+        response = make_api_request(base_url=self.client.base_url, endpoint=f"evaluations/{evaluation_id}", token=self.client.api_key)
+        return response
+    
+    def summarize(self, evaluation_id: str):
+        """
+        Return summary dataframe of the evaluation results, aggregated at every level
+        (overall eval, dimension, scenario, probe)
+
+        :param evaluation_id: The unique ID of the evaluation
+        :type evaluation_id: str
+        :return: A dataframe with the level, level_name, and score of the evaluation
+        :rtype: pandas.DataFrame
+        """
+
+        #initalize list of dicts where keys are level, level_name, score
+        summary_rows = []
+        
+        # get status of evaluation
+        try:
+            status_dict = self.get_status(evaluation_id=evaluation_id)
+        except ValueError as e: # for when evaluation doesn't exist
+            return e
+        
+        # evaluation level
+        if status_dict['status']=='COMPLETED':
+            row = dict()
+            row['level'] = 'overall'
+            row['level_name'] = 'evaluation'
+            row['score'] = round(status_dict['score']*100, 2)
+            summary_rows.append(row)            
+        else:
+            return ValueError("Evaluation is not completed yet, check back later.")
+        
+        # harness level
+        response = make_api_request(base_url=self.client.base_url, endpoint=f"harness?evaluation_id={evaluation_id}", token=self.client.api_key)
+        for harness in response['results']:
+            if harness['harness_type']=='DIMENSION' and harness['score'] is not None:
+                row = dict()
+                row['level'] = 'harness'
+                row['level_name'] = harness['name']
+                row['score'] = round(harness['score']*100, 2)
+                summary_rows.append(row)   
+                
+        # scenario level
+        response = make_api_request(base_url=self.client.base_url, endpoint=f"scenarios?evaluation_id={evaluation_id}", token=self.client.api_key)
+        for scenario in response['results']:
+            if scenario['score'] is not None:
+                row = dict()
+                row['level'] = 'scenario'
+                row['level_name'] = scenario['name']
+                row['score'] = round(scenario['score']*100, 2)
+                summary_rows.append(row)
+                
+        # probe level
+        response = make_api_request(base_url=self.client.base_url, endpoint=f"probes?evaluation_id={evaluation_id}", token=self.client.api_key)
+        for probe in response['results']:           
+            if probe['score'] is not None:
+                row = dict()
+                row['level'] = 'probe'
+                row['level_name'] = probe['name']
+                row['score'] = round(probe['score']*100, 2)
+                summary_rows.append(row)
+                
+        return pd.DataFrame(summary_rows)
+        
+    def export_summary(self, evaluation_id: str, output_dir: str = './', format: str='pdf'):
+        """
+        Export a PDF or HTML report of the evaluation results containing evaluation metadata
+        plus stats from summarize() function
+
+        :param evaluation_id: The unique ID of the evaluation
+        :type evaluation_id: str
+        :param output_dir: The directory to save the report to. Defaults to current directory
+        :type output_dir: str
+        :param format: The format of the report. Defaults to pdf. Options are 'pdf' and 'html'
+        :type format: str
+        :raises ValueError: If specified format is not 'pdf' or 'html'
+        """
+        #get dataframe of summary stats
+        summary = self.summarize(evaluation_id=evaluation_id)
+
+        #get evaluation metadata
+        status_dict = self.get_status(evaluation_id=evaluation_id)
+
+        #set jinja settings
+        HOME = os.path.join(os.path.dirname(__file__), ".")
+        templateLoader = jinja2.FileSystemLoader(
+            searchpath=HOME + "/templates"
+        )
+        templateEnv = jinja2.Environment(loader=templateLoader)
+        header_template = templateEnv.get_template("vijil_header.jinja")
+        footer_template = templateEnv.get_template("vijil_footer.jinja")
+        pdf_template = templateEnv.get_template("whole_pdf.jinja")
+
+        filename = evaluation_id + '_summary.' + format
+
+        data_fields = {
+            "reportfile": filename,
+            "run_uuid": evaluation_id,
+            "start_time": str(datetime.fromtimestamp(status_dict['created_at'], tz=timezone.utc)) + " UTC",
+            "created_by": status_dict['created_by'],
+            "end_time": str(datetime.fromtimestamp(status_dict['completed_at'], tz=timezone.utc)) + " UTC",
+            "model_name": status_dict["model"],
+            'total_test_count': status_dict['total_test_count'],
+            'completed_test_count': status_dict['completed_test_count'],
+            'error_test_count': status_dict['error_test_count'],
+            "results": summary.to_html(index=False)
+        }
+
+        #write to file
+        if format=='html': #use header and footer templates for html
+
+            # render the header.
+            digest_content = header_template.render(data_fields)            
+            # render the footer
+            digest_content += footer_template.render()
+
+            with open(os.path.join(output_dir, filename), "w") as f:
+                f.write(digest_content)
+
+        elif format=='pdf': #use whole_pdf template which leaves out CSS and JS
+            digest_content = pdf_template.render(data_fields)            
+            pdf = FPDF()
+            pdf.add_page()
+            pdf.write_html(digest_content)
+            pdf.output(os.path.join(output_dir, filename))
+        else:
+            raise ValueError("format must be 'html' or 'pdf'") 
+    
+    def describe(self, evaluation_id: str, limit: int = 1000, format: str = "dataframe", prettify: bool = True):
+        """
+        Return either list or dataframe of prompt-level metadata and evaluation results,
+        with metadata and evaluation scores for each prompt/response in the given evaluation id.
+
+        :param evaluation_id: The unique ID of the evaluation
+        :type evaluation_id: str
+        :param limit: The maximum number of prompts to include in description. Defaults to 1000.
+        :type limit: int, optional
+        :param format: The format of the output. Defaults to "dataframe". Options are "dataframe" and "list"
+        :type format: str, optional
+        :param prettify: If True, will remove the "vijil.probes." prefix from the probe names to make it more readable. Defaults to True.
+        :type prettify: bool, optional
+        :raises ValueError: If specified format is not 'dataframe' or 'list'
+        """
+        if limit is None:
+            limit = 1000000
+        
+        response = make_api_request(base_url=self.client.base_url, endpoint=f"responses?evaluation_id={evaluation_id}&limit={limit}", token=self.client.api_key)
+
+        # change detector ids to list if it is a string
+        results = response['results']
+        if len(results) == 0:
+            raise ValueError(f"No results found for evaluation id {evaluation_id}. Please check that you have the correct id.")
+        for idx, res in enumerate(results):
+            if isinstance(res['detector_ids'], str):
+                results[idx]['detector_ids'] = [res['detector_ids']]
+
+        prompt_list = [
+            {
+                "probe": res['probe_config_id'].replace("vijil.probes.", "") if prettify else res['probe_config_id'],
+                "input_prompt_id": res['input_prompt_id'],
+                "prompt": res['input_prompt'],
+                "response": res['response'],
+                "detectors": [
+                    det.replace("garak.detectors.", "").replace("autoredteam.detectors.", "")
+                    for det in res['detector_ids']
+                ],
+                "score": res['detector_scores'],
+                'generation': res['generation'],
+                'status': res['status'],
+                'error_message': res['error_message']
+            }
+            for res in results
+        ]
+
+        #calculate summary stats for detectors
+        prompt_list_df = pd.DataFrame(prompt_list)
+        prompt_list_df['avg_detector_score'] = prompt_list_df['score'].apply(lambda x: sum([sum(list(item.values())) for item in x]) / len(x) if len(x) and not any(['ERROR' in item for item in x]) else np.nan)
+        prompt_list_df['min_detector_score'] = prompt_list_df['score'].apply(lambda x: min([min(list(item.values())) for item in x]) / len(x) if len(x) and not any(['ERROR' in item for item in x]) else np.nan)
+        prompt_list_df['max_detector_score'] = prompt_list_df['score'].apply(lambda x: max([max(list(item.values())) for item in x]) / len(x) if len(x) and not any(['ERROR' in item for item in x]) else np.nan)
+        prompt_list_df.sort_values('probe', inplace=True)        
+
+        if format=='list':
+            return prompt_list_df.to_dict(orient='records')
+        elif format=='dataframe':
+            return prompt_list_df
+        else:
+            raise ValueError("format must be 'list' or 'dataframe'")
+
+    
+    def export_report(self, evaluation_id: str, limit: int=1000000, format: str='csv', output_dir: str = './'):
+        """
+        Exports output of describe() into csv, jsonl, json, or parquet 
+
+        :param evaluation_id: The unique ID of the evaluation
+        :type evaluation_id: str
+        :param limit: The maximum number of prompts to include in the report. Defaults to 1000000.
+        :type limit: int, optional
+        :param format: The format of the output. Defaults to "csv". Options are "csv", "parquet", "json" and "jsonl"
+        :type format: str, optional
+        :param output_dir: The directory to save the report. Defaults to the current directory.
+        :type output_dir: str, optional
+        :raises ValueError: If specified format is not 'csv', 'parquet', 'json' or 'jsonl'
+        """
+
+        if format=='csv':
+            prompt_list_df = self.describe(evaluation_id=evaluation_id, limit=limit, format="dataframe")
+            filename = evaluation_id + '_report.csv'
+            filepath = os.path.join(output_dir, filename)            
+            prompt_list_df.to_csv(filepath, index=False)
+            print("report exported to " + filepath)
+        elif format=='json':
+            prompt_list = self.describe(evaluation_id=evaluation_id, limit=limit, format="list")
+            filename = evaluation_id + '_report.json'
+            filepath = os.path.join(output_dir, filename)
+            with open(filepath, 'w') as f:
+                json.dump(prompt_list, f, indent=4)
+            print("report exported to " + filepath)
+        elif format=='jsonl':
+            prompt_list = self.describe(evaluation_id=evaluation_id, limit=limit, format="list")
+            filename = evaluation_id + '_report.jsonl'
+            filepath = os.path.join(output_dir, filename)
+            with open(filepath, 'w') as f:
+                for item in prompt_list:
+                    f.write(json.dumps(item) + '\n')
+            print("report exported to " + filepath)
+        elif format=='parquet':
+            prompt_list_df = self.describe(evaluation_id=evaluation_id, limit=limit, format="dataframe")
+            filename = evaluation_id + '_report.parquet'
+            filepath = os.path.join(output_dir, filename)
+            prompt_list_df.to_parquet(filepath, index=False)
+            print("report exported to " + filepath)
+        else:
+            raise ValueError("format must be 'csv', 'parquet', 'json', or 'jsonl'")
+
+    def cancel(self, evaluation_id: str):
+        """
+        Cancels an in-progress evaluation
+
+        :param evaluation_id: The unique ID of the evaluation
+        :type evaluation_id: str
+        :raises ValueError: If the evaluation is not in progress
+        :returns: The response from the API
+        :rtype: dict
+        """
+
+        #check eval status
+        try:
+            status_dict = self.get_status(evaluation_id=evaluation_id)
+        except ValueError as e:
+            return e
+        
+        if status_dict['status'] != 'IN_PROGRESS':
+            raise ValueError(f"Evaluation {evaluation_id} is not in progress. Cannot cancel.")
+        
+        payload = {
+                    "type": "CANCEL_EVALUATION",
+                    "data": {
+                        "evaluation_id": evaluation_id
+                    }
+                  }
+        
+        return make_api_request(
+            base_url=self.client.base_url,
+            endpoint="events",
+            method="post",
+            data=payload,
+            token=self.client.api_key
+        )
+    
+    def delete(self, evaluation_id: str):
+        """
+        Deletes an evaluation
+
+        :param evaluation_id: The unique ID of the evaluation
+        :type evaluation_id: str
+        :raises ValueError: If the evaluation does not exist
+        :returns: The response from the API
+        :rtype: dict
+        """
+
+        #check eval status. This will raise an error if the eval doesn't exist.
+        try:
+            status_dict = self.get_status(evaluation_id=evaluation_id)
+        except ValueError as e:
+            return e
+        
+        payload = {
+                    "type": "DELETE_EVALUATION",
+                    "data": {
+                        "evaluation_id": evaluation_id
+                    }
+                  }
+        
+        return make_api_request(
+            base_url=self.client.base_url,
+            endpoint="events",
+            method="post",
+            data=payload,
+            token=self.client.api_key
+        )
+
+class Vijil:
+    """Base class for the Vijil API client.
+    
+    :param base_url: The base URL for the Vijil API
+    :type base_url: str
+    :param api_key: The API key for the Vijil API
+    :type api_key: str
+    """
+    client: VijilClient
+    evaluations: Evaluations
+    harnesses: Harnesses
+    api_keys: APIKeys
+    
+    def __init__(self, base_url: str = BASE_URL, api_key: str = None):
+        """Constructor class for VijilClient
+
+        :param base_url: Base URL for API, defaults to BASE_URL as specified in `api.py`
+        :type base_url: str, optional
+        :param api_key: Vijil API key, defaults to None
+        :type api_key: str, optional
+        :raises ValueError: if no Vijil API key is provided in api_key and VIJIL_API_KEY is not set as an environment variable
+        """
+                
+        if api_key is None:
+            api_key = os.environ["VIJIL_API_KEY"]
+        if api_key is None:
+            raise ValueError(
+                "No API key found! Please set VIJIL_API_KEY as environment variable or supply the `api_key` parameter."
+            )
+            
+        self.client = VijilClient(base_url=base_url, api_key=api_key)
+        self.evaluations = Evaluations(self.client)
+        self.harnesses = Harnesses(self.client)
+        self.api_keys = APIKeys(self.client)

vijil-0.1.0/vijil/api.py

@@ -0,0 +1,62 @@
+# api_handler.py
+import requests
+import os
+import logging
+
+BASE_URL = "https://api-stage.vijil.ai/v1"
+
+SUPPORTED_HUBS = {
+    "openai": "OpenAI",
+    "together": "Together",
+    "octo": "OctoAI",
+}
+
+# Set up logging
+HIDDEN_DIR_NAME = ".vijil"
+LOG_FILE_NAME = "vijil.log"
+LOG_FILE_PATH = os.path.join(os.path.expanduser('~'), HIDDEN_DIR_NAME, LOG_FILE_NAME)
+os.makedirs(os.path.dirname(LOG_FILE_PATH), exist_ok=True)
+logging.basicConfig(level=logging.ERROR, filename=LOG_FILE_PATH, filemode='a',
+                    format='%(asctime)s - %(levelname)s - %(message)s')
+logger = logging.getLogger(__name__)
+
+
+def make_api_request(base_url, endpoint, method="get", token=None, params=None, data=None):
+    url = f"{base_url}/{endpoint}"
+    if token is not None:
+        headers = {"Authorization": f"Bearer {token}"}
+    else:
+        headers = {}
+            
+    try:
+        if method.lower() == "get":
+            response = requests.get(url, params=params, headers=headers)
+            # print("GET request made") # code to test if the request is being made
+        elif method.lower()=='put':
+            response = requests.put(url, json=data, params=params, headers=headers)
+        elif method.lower() == "delete":
+            response = requests.delete(url, params=params, headers=headers)
+        elif method.lower() == "post":
+            response = requests.post(url, json=data, params=params, headers=headers)
+            # print(f"POST request made to {url} with payload {data}") # code to test if the request is being made
+        else:
+            raise ValueError("Invalid HTTP method. Use 'get', 'put', 'delete', or 'post'.")
+
+        response.raise_for_status()
+        return response.json()
+    except:
+        print(f"Error: {response.text}")
+        pass
+
+def get_api_proxy_dict(base_url, token=None):
+
+    response = make_api_request(base_url=base_url, endpoint="api-keys", token=token)
+    
+    unique_hubs = set([r['hub'] for r in response])
+    hubs_dict = {hub: {} for hub in unique_hubs}
+    for hub in unique_hubs:
+        for r in response:
+            if r['hub'] == hub:
+                hubs_dict[hub][r['name']] = r['id']
+                
+    return hubs_dict

vijil-0.1.0/vijil/templates/vijil_footer.jinja

@@ -0,0 +1,27 @@
+
+<script>
+var acc = document.getElementsByClassName("accordion");
+var i;
+
+for (i = 0; i < acc.length; i++) {
+  acc[i].addEventListener("click", function() {
+    /* Toggle between adding and removing the "active" class,
+    to highlight the button that controls the panel */
+    this.classList.toggle("active");
+
+    /* Toggle between hiding and showing the active panel */
+    var panel = this.nextElementSibling;
+    if (panel.style.display === "block") {
+      panel.style.display = "none";
+    } else {
+      panel.style.display = "block";
+    }
+  });
+}</script>
+</body>
+
+<p style="height:400px; padding-top: 100px">
+generated with <a href="https://vijil.ai">Vijil</a>
+</p>
+
+</html>

vijil-0.1.0/vijil/templates/vijil_header.jinja

@@ -0,0 +1,79 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<meta charset="UTF-8" />
+<style>
+body {font-family: sans-serif}
+.defcon1 {background-color: #f94144; text-color: #000}
+.defcon2 {background-color: #f8961e; text-color: #000}
+.defcon3 {background-color: #ccc; text-color: #000}
+.defcon4 {background-color: #ddd; text-color: #000}
+.defcon5 {background-color: #f7f7f7; text-color: #000}
+.probe {padding-left: 40pt}
+.detector {padding-left: 65pt}
+h2 {padding-left: 20pt}
+h3 {padding-left: 40pt}
+h4 {padding-left: 60pt}
+h2,h3,h4 {padding-top: 10px; padding-bottom: 10px}
+th, td {
+  padding: 5px;
+  text-align: left;
+}
+tr {
+  border-bottom: 1px solid #ddd;
+}
+
+/* Style the buttons that are used to open and close the accordion panel */
+.accordion {
+//  background-color: #eee;
+  color: #444;
+  cursor: pointer;
+  padding: 18px;
+  width: 100%;
+  text-align: left;
+  border: none;
+  outline: none;
+  transition: 0.4s;
+  margin: 1pt;
+}
+
+/* Add a background color to the button if it is clicked on (add the .active class with JS), and when you move the mouse over it (hover) */
+.active, .accordion:hover {
+  background-color: #ccf;
+}
+
+/* Style the accordion panel. Note: hidden by default */
+.panel {
+  padding: 0 18px;
+  background-color: white;
+  display: none;
+  overflow: hidden;
+}
+</style>
+<title>Trust Report: {{model_name}}</title>
+<meta name="description" content="" />
+</head>
+
+<body>
+
+<h1>Trust Report: {{model_name}}</h1>
+<button class="accordion">⚙ Config</button>
+<div style="border:solid black 1px; padding: 5px; margin: 5px" class="panel">
+<h2>Config</h2>
+<p>Filename: {{reportfile}}</p>
+<p>Evaluation ID: {{run_uuid}}</p>
+<p>Model name: {{model_name}}</p>
+<p>Evaluation started: {{start_time}}</p>
+<p>Evaluation ended: {{end_time}}</p>
+<p>Created by: {{created_by}}</p>
+<p>Total tests: {{total_test_count}}</p>
+<p>Completed tests: {{completed_test_count}}</p>
+<p>Tests with errors: {{error_test_count}}</p>
+</div>
+
+<h3>Model name: {{model_name}}</h3>
+<h3>Evaluation Time: {{start_time}}</h3>
+
+{{ results | safe}}

vijil-0.1.0/vijil/templates/whole_pdf.jinja

@@ -0,0 +1,35 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<meta charset="UTF-8" />
+</head>
+
+<body>
+
+<h1>Trust Report: {{model_name}}</h1>
+<div style="border:solid black 1px; padding: 5px; margin: 5px" class="panel">
+<h2>Configuration details</h2>
+<p>Filename: {{reportfile}}</p>
+<p>Evaluation ID: {{run_uuid}}</p>
+<p>Model name: {{model_name}}</p>
+<p>Evaluation started: {{start_time}}</p>
+<p>Evaluation ended: {{end_time}}</p>
+<p>Created by: {{created_by}}</p>
+<p>Total tests: {{total_test_count}}</p>
+<p>Completed tests: {{completed_test_count}}</p>
+<p>Tests with errors: {{error_test_count}}</p>
+</div>
+
+<br>
+
+{{ results | safe}}
+
+</body>
+
+<p style="height:400px; padding-top: 100px">
+generated with <a href="https://vijil.ai">Vijil</a>
+</p>
+
+</html>

vijil-0.1.0/vijil/types.py

@@ -0,0 +1,9 @@
+from dataclasses import dataclass
+
+from vijil.api import BASE_URL
+
+
+@dataclass
+class VijilClient:
+    base_url: str = BASE_URL
+    api_key: str = None