arthur-client 1.3.0__py3-none-any.whl → 1.4.1055__py3-none-any.whl

arthur/client/rest/bench/client.py

@@ -1,280 +0,0 @@
-from typing import Optional, Any, Union, Dict, List, Tuple
-from http import HTTPStatus
-from requests.cookies import RequestsCookieJar
-from requests import Response
-
-# import http client
-from arthur.client.http.requests import HTTPClient
-
-from arthur.client.rest.bench.models import (
-    CreateRunRequest,
-    CreateRunResponse,
-    HallucinationCheckRequest,
-    HallucinationCheckResponse,
-    PaginatedGetRunResponse,
-    PaginatedGetRunsForTestSuiteResponse,
-    PaginatedGetTestSuiteResponse,
-    PaginatedGetTestSuitesResponse,
-    TestSuiteRequest,
-    TestSuiteResponse,
-    TestSuiteSummaryResponse,
-)
-
-
-PATH_PREFIX = "/api"
-
-
-class ArthurBenchClient:
-    """
-    A Python client to interact with the Arthur Bench API
-    """
-
-    def __init__(self, http_client: HTTPClient):
-        """
-        Create a new ArthurBenchClient from an HTTPClient
-
-        :param http_client: the :class:`~arthurai.client.http.requests.HTTPClient` to use for underlying requests
-        """
-        self.http_client = http_client
-        self.http_client.set_path_prefix(PATH_PREFIX)
-
-    def get_test_suites(
-        self,
-        name: Optional[str] = None,
-        sort: Optional[str] = None,
-        page: Optional[int] = None,
-        page_size: Optional[int] = None,
-    ) -> PaginatedGetTestSuitesResponse:
-        """
-                Gets test suites
-
-                Sort by latest run by default.
-        If `name` query parameter is provided, filter on test suite name.
-
-                :param name:
-                :param sort:
-                :param page:
-                :param page_size:
-        """
-
-        params: Dict[str, Any] = {}
-        if name is not None:
-            params["name"] = name
-        if sort is not None:
-            params["sort"] = sort
-        if page is not None:
-            params["page"] = page
-        if page_size is not None:
-            params["page_size"] = page_size
-
-        parsed_resp: Dict[str, Any] = self.http_client.get(  # type: ignore
-            f"/v3/bench/test_suites", params=params, validation_response_code=200
-        )
-        return PaginatedGetTestSuitesResponse(**parsed_resp)
-
-    def create_test_suite(self, json_body: TestSuiteRequest) -> TestSuiteResponse:
-        """
-        Creates a new test suite from reference data using specified scoring_method for scoring
-
-        :param json_body:
-        """
-
-        parsed_resp: Dict[str, Any] = self.http_client.post(  # type: ignore
-            f"/v3/bench/test_suites",
-            json=json_body.dict(by_alias=True, exclude_none=True),
-            validation_response_code=201,
-        )
-        return TestSuiteResponse(**parsed_resp)
-
-    def get_test_suite(
-        self,
-        test_suite_id: str,
-        page: Optional[int] = None,
-        page_size: Optional[int] = None,
-    ) -> PaginatedGetTestSuiteResponse:
-        """
-        Get reference data for an existing test suite
-
-        :param test_suite_id:
-        :param page:
-        :param page_size:
-        """
-
-        params: Dict[str, Any] = {}
-        if page is not None:
-            params["page"] = page
-        if page_size is not None:
-            params["page_size"] = page_size
-
-        parsed_resp: Dict[str, Any] = self.http_client.get(  # type: ignore
-            f"/v3/bench/test_suites/{test_suite_id}",
-            params=params,
-            validation_response_code=200,
-        )
-        return PaginatedGetTestSuiteResponse(**parsed_resp)
-
-    def delete_test_suite(self, test_suite_id: str) -> Response:
-        """
-        Deletes test suite
-
-        Is idempotent.
-
-        :param test_suite_id:
-        """
-
-        raw_resp: Response = self.http_client.delete(  # type: ignore
-            f"/v3/bench/test_suites/{test_suite_id}",
-            validation_response_code=204,
-            return_raw_response=True,
-        )
-        return raw_resp
-
-    def get_summary_statistics(
-        self,
-        test_suite_id: str,
-        run_ids: Optional[List[str]] = None,
-        page: Optional[int] = None,
-        page_size: Optional[int] = None,
-    ) -> TestSuiteSummaryResponse:
-        """
-        Get paginated summary statistics of a test suite
-
-        Defaults to page size of 5.
-
-        :param test_suite_id:
-        :param run_ids:
-        :param page:
-        :param page_size:
-        """
-
-        params: Dict[str, Any] = {}
-        if run_ids is not None:
-            params["run_ids"] = run_ids
-        if page is not None:
-            params["page"] = page
-        if page_size is not None:
-            params["page_size"] = page_size
-
-        parsed_resp: Dict[str, Any] = self.http_client.get(  # type: ignore
-            f"/v3/bench/test_suites/{test_suite_id}/runs/summary",
-            params=params,
-            validation_response_code=200,
-        )
-        return TestSuiteSummaryResponse(**parsed_resp)
-
-    def get_runs_for_test_suite(
-        self,
-        test_suite_id: str,
-        sort: Optional[str] = None,
-        page: Optional[int] = None,
-        page_size: Optional[int] = None,
-    ) -> PaginatedGetRunsForTestSuiteResponse:
-        """
-        Get runs for a particular test suite (identified by test_suite_id)
-
-        :param test_suite_id:
-        :param sort:
-        :param page:
-        :param page_size:
-        """
-
-        params: Dict[str, Any] = {}
-        if sort is not None:
-            params["sort"] = sort
-        if page is not None:
-            params["page"] = page
-        if page_size is not None:
-            params["page_size"] = page_size
-
-        parsed_resp: Dict[str, Any] = self.http_client.get(  # type: ignore
-            f"/v3/bench/test_suites/{test_suite_id}/runs",
-            params=params,
-            validation_response_code=200,
-        )
-        return PaginatedGetRunsForTestSuiteResponse(**parsed_resp)
-
-    def create_new_test_run(
-        self, test_suite_id: str, json_body: CreateRunRequest
-    ) -> CreateRunResponse:
-        """
-        Creates a new test run with model version / associated metadata
-
-
-        :param test_suite_id:
-        :param json_body:
-        """
-
-        parsed_resp: Dict[str, Any] = self.http_client.post(  # type: ignore
-            f"/v3/bench/test_suites/{test_suite_id}/runs",
-            json=json_body.dict(by_alias=True, exclude_none=True),
-            validation_response_code=201,
-        )
-        return CreateRunResponse(**parsed_resp)
-
-    def get_test_run(
-        self,
-        test_suite_id: str,
-        test_run_id: str,
-        page: Optional[int] = None,
-        page_size: Optional[int] = None,
-        sort: Optional[bool] = None,
-    ) -> PaginatedGetRunResponse:
-        """
-        Get a test run with input, output, and reference data
-
-        :param test_suite_id:
-        :param test_run_id:
-        :param page:
-        :param page_size:
-        :param sort:
-        """
-
-        params: Dict[str, Any] = {}
-        if page is not None:
-            params["page"] = page
-        if page_size is not None:
-            params["page_size"] = page_size
-        if sort is not None:
-            params["sort"] = sort
-
-        parsed_resp: Dict[str, Any] = self.http_client.get(  # type: ignore
-            f"/v3/bench/test_suites/{test_suite_id}/runs/{test_run_id}",
-            params=params,
-            validation_response_code=200,
-        )
-        return PaginatedGetRunResponse(**parsed_resp)
-
-    def delete_a_test_run(self, test_suite_id: str, test_run_id: str) -> Response:
-        """
-        Deletes a test run
-
-        Is idempotent.
-
-        :param test_suite_id:
-        :param test_run_id:
-        """
-
-        raw_resp: Response = self.http_client.delete(  # type: ignore
-            f"/v3/bench/test_suites/{test_suite_id}/runs/{test_run_id}",
-            validation_response_code=204,
-            return_raw_response=True,
-        )
-        return raw_resp
-
-    def score_a_response_for_hallucination_given_a_context(
-        self, json_body: HallucinationCheckRequest
-    ) -> HallucinationCheckResponse:
-        """
-        Score a response for hallucination given a context
-
-        Return True if the response contains a hallucination and False otherwise
-
-        :param json_body: Model generated response and context used to determine whether the response is a hallucination
-        """
-
-        parsed_resp: Dict[str, Any] = self.http_client.post(  # type: ignore
-            f"/v3/bench/scoring/hallucination",
-            json=json_body.dict(by_alias=True, exclude_none=True),
-            validation_response_code=200,
-        )
-        return HallucinationCheckResponse(**parsed_resp)

arthur/client/rest/bench/models.py

@@ -1,229 +0,0 @@
-# generated by datamodel-codegen:
-#   filename:  bench.yaml
-#   timestamp: 2024-01-24T16:44:22+00:00
-
-from __future__ import annotations
-
-from datetime import datetime
-from typing import List, Optional
-from uuid import UUID
-
-from arthur.client.rest.common.models import Page, PageSize, TotalCount, TotalPages
-from pydantic import BaseModel, Field
-
-
-class TestCaseRequest(BaseModel):
-    """
-    An input, reference output pair.
-    """
-
-    input: str
-    """
-    Input to the test case. Does not include the prompt template.
-    """
-    reference_output: Optional[str] = None
-    """
-    Reference or "Golden" output for the given input.
-    """
-
-
-class TestCaseResponseItem(BaseModel):
-    id: Optional[UUID] = None
-    input: Optional[str] = None
-    """
-    Input to the test case. Does not include the prompt template.
-    """
-    reference_output: Optional[str] = None
-    """
-    Reference or "Golden" output for the given input.
-    """
-
-
-class TestCaseResponse(BaseModel):
-    """
-    A test case (input, reference output pair).
-    """
-
-    __root__: List[TestCaseResponseItem]
-    """
-    A test case (input, reference output pair).
-    """
-
-
-class HistogramItem(BaseModel):
-    count: int
-    low: float
-    high: float
-
-
-class SummaryItem(BaseModel):
-    name: str
-    id: UUID
-    avg_score: float
-    histogram: List[HistogramItem]
-
-
-class TestSuiteSummaryResponse(BaseModel):
-    num_test_cases: float
-    summary: List[SummaryItem]
-    page: Page
-    page_size: PageSize
-    total_pages: TotalPages
-    total_count: TotalCount
-
-
-class TestCaseOutput(BaseModel):
-    id: Optional[UUID] = None
-    output: Optional[str] = None
-    score: Optional[float] = None
-
-
-class CreateRunRequest(BaseModel):
-    test_case_outputs: List[TestCaseOutput]
-    description: Optional[str] = None
-    model_name: Optional[str] = None
-    name: str
-    foundation_model: Optional[str] = None
-    prompt_template: Optional[str] = None
-    model_version: Optional[str] = None
-
-
-class CreateRunResponse(BaseModel):
-    id: Optional[UUID] = None
-
-
-class RunResult(BaseModel):
-    id: str
-    input: str
-    reference_output: Optional[str] = None
-    output: str
-    score: float
-
-
-class TestRun(BaseModel):
-    id: Optional[UUID] = None
-    name: Optional[str] = None
-    created_at: Optional[datetime] = None
-    updated_at: Optional[datetime] = None
-    model_version: Optional[str] = None
-    prompt_template: Optional[str] = None
-    avg_score: Optional[float] = None
-
-
-class PaginatedGetRunsForTestSuiteResponse(BaseModel):
-    """
-    Paginated list of runs for a test suite.
-    """
-
-    test_runs: List[TestRun]
-    page: Page
-    page_size: PageSize
-    total_pages: TotalPages
-    total_count: TotalCount
-
-
-class ScoringMethod(BaseModel):
-    name: str
-    type: str
-
-
-class PaginatedGetRunResponse(BaseModel):
-    """
-    Paginated list of prompts, reference outputs, and model outputs for a particular run.
-    """
-
-    id: UUID
-    created_at: datetime
-    updated_at: datetime
-    description: Optional[str] = None
-    name: Optional[str] = None
-    metadata: Optional[str] = None
-    test_case_runs: List[RunResult]
-    test_suite_id: UUID
-    page: Page
-    page_size: PageSize
-    total_pages: Optional[TotalPages] = None
-    total_count: Optional[TotalCount] = None
-
-
-class HallucinationCheckRequest(BaseModel):
-    """
-    Model generated response and context used to determine whether the response is a hallucination
-    """
-
-    response: str
-    """
-    model generated response string
-    """
-    context: str
-    """
-    string providing context in which model generated response should be consistent with
-    """
-
-
-class HallucinationCheckResponse(BaseModel):
-    """
-    Binary hallucination classification and reasoning for classification
-    """
-
-    hallucination: bool
-    """
-    if the response contained a hallucination
-    """
-    reason: str
-    """
-    justification of the hallucination classification
-    """
-
-
-class TestSuite(BaseModel):
-    id: UUID
-    name: str
-    scoring_method: ScoringMethod
-    last_run_time: Optional[datetime] = None
-    description: Optional[str] = None
-    created_at: Optional[datetime] = None
-    updated_at: Optional[datetime] = None
-
-
-class PaginatedGetTestSuitesResponse(BaseModel):
-    test_suites: List[TestSuite]
-    page: Page
-    page_size: PageSize
-    total_pages: TotalPages
-    total_count: TotalCount
-
-
-class TestSuiteRequest(BaseModel):
-    name: str
-    description: Optional[str] = None
-    scoring_method: ScoringMethod
-    test_cases: List[TestCaseRequest] = Field(..., min_items=1)
-    """
-    JSON object containing test case data for the test suite.
-    """
-
-
-class TestSuiteResponse(BaseModel):
-    id: UUID
-    name: str
-    description: Optional[str] = None
-    scoring_method: ScoringMethod
-    organization_id: Optional[UUID] = None
-    created_at: datetime
-    updated_at: datetime
-    test_cases: List[TestCaseResponse] = Field(..., min_items=1)
-    """
-    JSON object containing test case data for the test suite.
-    """
-
-
-class PaginatedGetTestSuiteResponse(BaseModel):
-    id: UUID
-    name: str
-    scoring_method: ScoringMethod
-    test_cases: List[TestCaseResponse]
-    page: Optional[Page] = None
-    page_size: Optional[PageSize] = None
-    total_pages: Optional[TotalPages] = None
-    total_count: Optional[TotalCount] = None

arthur/client/rest/client.py

@@ -1,158 +0,0 @@
-import logging
-import os
-from distutils.util import strtobool
-from getpass import getpass
-from platform import system
-from typing import Optional
-
-from arthur.client.auth.helpers import get_current_org, user_login
-from arthur.client.auth.refresh import AuthRefresher
-from arthur.client.http.requests import HTTPClient
-from arthur.common.exceptions import MissingParameterError, UserValueError
-from arthur.client.version import __version__
-
-# import sub-clients
-from arthur.client.rest.admin.client import ArthurAdminClient
-from arthur.client.rest.alerts.client import ArthurAlertsClient
-from arthur.client.rest.bench.client import ArthurBenchClient
-from arthur.client.rest.common.client import ArthurClientCommon
-from arthur.client.rest.enrichments.client import ArthurEnrichmentsClient
-from arthur.client.rest.inferences.client import ArthurInferencesClient
-from arthur.client.rest.insights.client import ArthurInsightsClient
-from arthur.client.rest.metrics.client import ArthurMetricsClient
-from arthur.client.rest.model_groups.client import ArthurModelGroupsClient
-from arthur.client.rest.models.client import ArthurModelsClient
-from arthur.client.rest.query.client import ArthurQueryClient
-from arthur.client.rest.users.client import ArthurUsersClient
-# end import sub-clients
-
-UNKNOWN_ORG_ID = "unknown-org"
-ORG_ID_HEADER = "Arthur-Organization-ID"
-
-
-logger = logging.getLogger(__name__)
-
-
-class ArthurClient:
-    def __init__(
-        self,
-        url: Optional[str] = None,
-        login: Optional[str] = None,
-        password: Optional[str] = None,
-        api_key: Optional[str] = None,
-        organization_id: Optional[str] = None,
-        verify_ssl: Optional[bool] = None,
-        allow_insecure: bool = False,
-        offline: bool = False,
-    ):
-        """
-        TODO!
-        """
-        # basic values
-        if url is None:
-            url = os.getenv("ARTHUR_API_URL")
-        if url is None:
-            raise MissingParameterError(
-                "You must provide a URL through the 'url' parameter or the ARTHUR_API_URL "
-                "environment variable"
-            )
-        if verify_ssl is None:  # if not provided
-            # read from env var, or default to True if not present
-            try:
-                verify_ssl = bool(strtobool(os.getenv("ARTHUR_VERIFY_SSL", "true")))
-            except ValueError:
-                raise UserValueError(
-                    f"ARTHUR_VERIFY_SSL environment variable must be a boolean value, got "
-                    f"{os.getenv('ARTHUR_VERIFY_SSL')}"
-                )
-
-        # authorization
-        # TODO: allow specified login/pass to override an env var api key
-        if login is None:
-            login = os.getenv("ARTHUR_LOGIN")
-        if password is None:
-            password = os.getenv("ARTHUR_PASSWORD")
-        if api_key is None and login is None:
-            api_key = os.getenv("ARTHUR_API_KEY")
-        # validate only login or api key
-        if login is not None and api_key is not None:
-            raise UserValueError(
-                "You may not provide both a login and api key, please ensure you are are supplying only "
-                "one through the login/api_key parameters and ARTHUR_LOGIN/ARTHUR_API_KEY environment "
-                "variables"
-            )
-        if login is not None:  # login if provided
-            # if password not supplied, get it from input
-            if password is None:
-                password = getpass(f"Please enter password for {login}: ")
-
-            # Get session token from login and password
-            auth_token = user_login(
-                api_http_host=url, login=login, password=password, verify_ssl=verify_ssl
-            )
-            # create an auth refresher
-            auth_refresher = AuthRefresher(
-                url=url, login=login, password=password, verify_ssl=verify_ssl
-            )
-            header_refresh_func = auth_refresher.refresh
-        elif api_key is not None:  # if api key provided, set that
-            auth_token = api_key
-            header_refresh_func = None
-        else:
-            raise MissingParameterError(
-                "No authentication provided. Please supply a login (username or email) through "
-                "the 'login' parameter or ARTHUR_LOGIN environment variable.\n\nIf this is a "
-                "production environment, alternatively consider providing an API key through "
-                "the 'api_key' parameter or ARTHUR_API_KEY environment variable."
-            )
-
-        # org id
-        if offline:
-            if organization_id is not None:
-                raise UserValueError(
-                    "You cannot specify an organization ID if you are offline."
-                )
-            else:
-                organization_id = UNKNOWN_ORG_ID
-        else:  # if online
-            # fill org ID with environment variable if not provided
-            if organization_id is None:
-                organization_id = os.getenv("ARTHUR_ORGANIZATION_ID")
-            # if still no org ID, fetch it from the API
-            if organization_id is None:
-                organization_id = get_current_org(
-                    url, auth_token, verify_ssl=verify_ssl
-                )
-
-        # TODO: consider having the SDK override this?
-        user_agent = f"arthur-client/{__version__} (system={system()})"
-        headers = {
-            "Accept": "application/json",
-            "Authorization": auth_token,
-            "User-Agent": user_agent,
-            ORG_ID_HEADER: organization_id,
-        }
-
-        # setup http client construction arguments
-        client_kwargs = {
-            "base_url": url,
-            "default_headers": headers,
-            "verify_ssl": verify_ssl,
-            "allow_insecure": allow_insecure,
-            "header_refresh_func": header_refresh_func,
-        }
-
-        # create client object for each client subcomponent
-        self.admin = ArthurAdminClient(HTTPClient(**client_kwargs))  # type: ignore
-        self.alerts = ArthurAlertsClient(HTTPClient(**client_kwargs))  # type: ignore
-        self.bench = ArthurBenchClient(HTTPClient(**client_kwargs))  # type: ignore
-        self.common = ArthurClientCommon(HTTPClient(**client_kwargs))  # type: ignore
-        self.enrichments = ArthurEnrichmentsClient(HTTPClient(**client_kwargs))  # type: ignore
-        self.inferences = ArthurInferencesClient(HTTPClient(**client_kwargs))  # type: ignore
-        self.insights = ArthurInsightsClient(HTTPClient(**client_kwargs))  # type: ignore
-        self.metrics = ArthurMetricsClient(HTTPClient(**client_kwargs))  # type: ignore
-        self.model_groups = ArthurModelGroupsClient(HTTPClient(**client_kwargs))  # type: ignore
-        self.models = ArthurModelsClient(HTTPClient(**client_kwargs))  # type: ignore
-        self.query = ArthurQueryClient(HTTPClient(**client_kwargs))  # type: ignore
-        self.users = ArthurUsersClient(HTTPClient(**client_kwargs))  # type: ignore
-        # end client object creation

arthur/client/rest/common/__init__.py

@@ -1,2 +0,0 @@
-from . import models
-from . import client

arthur/client/rest/common/client.py

@@ -1,25 +0,0 @@
-from typing import Optional, Any, Union, Dict, List, Tuple
-from http import HTTPStatus
-from requests.cookies import RequestsCookieJar
-from requests import Response
-
-# import http client
-from arthur.client.http.requests import HTTPClient
-
-
-PATH_PREFIX = "/"
-
-
-class ArthurClientCommon:
-    """
-    A Python client to interact with the Arthur API Common
-    """
-
-    def __init__(self, http_client: HTTPClient):
-        """
-        Create a new ArthurClientCommon from an HTTPClient
-
-        :param http_client: the :class:`~arthurai.client.http.requests.HTTPClient` to use for underlying requests
-        """
-        self.http_client = http_client
-        self.http_client.set_path_prefix(PATH_PREFIX)