codemie-sdk-python 0.1.73__py3-none-any.whl → 0.1.75__py3-none-any.whl

codemie_sdk/models/assistant.py

@@ -263,3 +263,18 @@ class EnvVars(BaseModel):
 
 class ExportAssistantPayload(BaseModel):
     env_vars: Optional[EnvVars] = None
+
+
+class AssistantEvaluationRequest(BaseModel):
+    """Model for assistant evaluation request."""
+
+    model_config = ConfigDict(extra="ignore")
+
+    dataset_id: str = Field(description="ID of the dataset to use for evaluation")
+    experiment_name: str = Field(description="Name of the evaluation experiment")
+    system_prompt: Optional[str] = Field(
+        default=None, description="System prompt to use for evaluation"
+    )
+    llm_model: Optional[str] = Field(
+        default=None, description="LLM model to use for evaluation"
+    )

codemie_sdk/services/assistant.py

@@ -16,6 +16,7 @@ from ..models.assistant import (
     AssistantBase,
     Context,
     ExportAssistantPayload,
+    AssistantEvaluationRequest,
 )
 from ..models.common import PaginationParams
 from ..utils import ApiRequestHandler
@@ -37,7 +38,7 @@ class AssistantService:
     def list(
         self,
         minimal_response: bool = True,
-        scope: Literal["visible_to_user", "created_by_user"] = "visible_to_user",
+        scope: Literal["visible_to_user", "marketplace"] = "visible_to_user",
         page: int = 0,
         per_page: int = 12,
         filters: Optional[Dict[str, Any]] = None,
@@ -213,3 +214,19 @@ class AssistantService:
             stream=True,
             json_data=request.model_dump(exclude_none=True),
         )
+
+    def evaluate(self, assistant_id: str, request: AssistantEvaluationRequest) -> dict:
+        """Evaluate an assistant with a dataset.
+
+        Args:
+            assistant_id: ID of the assistant to evaluate
+            request: Evaluation request details
+
+        Returns:
+            Evaluation results
+        """
+        return self._api.post(
+            f"/v1/assistants/{assistant_id}/evaluate",
+            dict,
+            json_data=request.model_dump(exclude_none=True),
+        )

{codemie_sdk_python-0.1.73.dist-info → codemie_sdk_python-0.1.75.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: codemie-sdk-python
-Version: 0.1.73
+Version: 0.1.75
 Summary: CodeMie SDK for Python
 Author: Vadym Vlasenko
 Author-email: vadym_vlasenko@epam.com
@@ -746,33 +746,39 @@ For providing credentials please contact AI/Run CodeMie Team: Vadym_Vlasenko@epa
 
 ## Running tests
 
-For running tests on custom environment you should create .env file in the /tests directory,
-ask QA team: anton_yeromin@epam.com to provide all needed testing credentials
+For running tests on custom environment you should create .env file in the ./tests directory,
+ask QA team: anton_yeromin@epam.com to provide all needed testing credentials. Under this directory there are stubs
+for .env files for running tests on local and preview environments.
+
+Configuration example:
 
 ``` properties
 
 AUTH_SERVER_URL=https://keycloak.eks-core.aws.main.edp.projects.epam.com/auth
 AUTH_CLIENT_ID=codemie-preview
-AUTH_CLIENT_SECRET=<your_secret>
+AUTH_CLIENT_SECRET=<auth_clienbt_secret>
 AUTH_REALM_NAME=codemie-prod
-CODEMIE_API_DOMAIN=https://codemie-preview.lab.epam.com/code-assistant-api
-NATS_URL=nats://nats-preview-codemie.epmd-edp-anthos.eu.gcp.cloudapp.epam.com:443
+CODEMIE_API_DOMAIN=http://localhost:8080
 VERIFY_SSL=False
 
-USERNAME=<username>
-PASSWORD=<password>
+NATS_URL=nats://localhost:4222
 
-TEST_USER_FULL_NAME="Test User"
+ENV=local
 CLEANUP_DATA=True
 
-PROJECT_NAME=automation-tests-project
+AUTH_USERNAME=<username>
+AUTH_PASSWORD=<password>
+TEST_USER_FULL_NAME=<user_full_name>
+
+PROJECT_NAME=codemie
+GIT_ENV=gitlab
 
 DEFAULT_TIMEOUT=60
 
 GITLAB_URL=https://gitbud.epam.com
 GITLAB_TOKEN=<gitlab_token>
 GITLAB_PROJECT=https://gitbud.epam.com/epm-cdme/autotests/codemie-test-project
-GITLAB_PROJECT_ID=<project_id>
+GITLAB_PROJECT_ID=17889
 
 GITHUB_URL=https://github.com
 GITHUB_TOKEN=<github_token>
@@ -786,9 +792,8 @@ CONFLUENCE_URL=https://kb.epam.com
 CONFLUENCE_TOKEN=<confluence_token>
 CQL="space = EPMCDME and type = page and title = 'AQA Backlog Estimation'"
 
-AWS_ACCESS_KEY=<aws_access_key>
+AWS_ACCESS_KEY=<aws_access_token>
 AWS_SECRET_KEY=<aws_secret_key>
-AWS_SESSION_TOKEN=<aws_session_token>
 ```
 
 Run all tests
@@ -797,10 +802,10 @@ Run all tests
 pytest -n auto --reruns 1
 ```
 
-Run e2e tests
+Run e2e/regression tests
 
 ```shell
-pytest -n auto -m e2e --reruns 1
+pytest -n auto -m "e2e or regression" --reruns 1
 ```
 
 Run tests for e2e tests for specific integration/tool.

{codemie_sdk_python-0.1.73.dist-info → codemie_sdk_python-0.1.75.dist-info}/RECORD

@@ -4,7 +4,8 @@ codemie_sdk/auth/credentials.py,sha256=u2eLNsD8fELTgreQghVb0g0kk82zchUkux0M5lzq-
 codemie_sdk/client/__init__.py,sha256=yf6C39MmrJ6gK9ZHMhBeynKwUUYVSUTQbKxU8-4qpKg,101
 codemie_sdk/client/client.py,sha256=dVcWqd-ruWd8GOZ3_33C3LnHgUyyxIoCKxKYPYbBPq8,4373
 codemie_sdk/exceptions.py,sha256=XoVPyognx-JmyVxLHkZPAcX1CMi1OoT1diBFJLU54so,1183
-codemie_sdk/models/assistant.py,sha256=U0g-DFiybxz7gBvtuEcSeiT0WXz-VZY__9dE12QNMFE,7982
+codemie_sdk/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+codemie_sdk/models/assistant.py,sha256=ZEhn5xH0EsmmZMI2U8kaGryPfFgg-YzlIV9XmOFiUMs,8524
 codemie_sdk/models/common.py,sha256=gmZ-ps8TbaieNKr0kUKoQEjhVrHD2CAYomOpZQRatH8,1195
 codemie_sdk/models/datasource.py,sha256=u6X1xXZ3ZR9CWH0B0bSw3IjC7KyEwtRKHIc6ZuhJwsg,10813
 codemie_sdk/models/integration.py,sha256=c2D7QsM9HJ9zpupkjKz1v60rtBMGgTyJLYXUyAzS5no,2240
@@ -13,7 +14,7 @@ codemie_sdk/models/task.py,sha256=J4ZFRY3s8qBGrqB5NLQF0rMbInLh4s7OEZ0ZfmnW0Ho,14
 codemie_sdk/models/user.py,sha256=Q0rjimZh-IbeaPfq6b6fk6ZaCtwLqWHEIlU863suCS4,1777
 codemie_sdk/models/workflow.py,sha256=h3ZM-2bBucRd9CzZUYDTU0rGR1ihEN7cxkHgQmJfJVc,2394
 codemie_sdk/models/workflow_state.py,sha256=CMYFQZ7sy4QxmnWmc83TFfqP7TG_3rW5MdH5fxsS9kY,1251
-codemie_sdk/services/assistant.py,sha256=xuzdel93VlOOy_xCmuptlwpChQWOvuPbpBoB2nF-Sx0,6499
+codemie_sdk/services/assistant.py,sha256=0OFx7enE65I2uiLpbIebnCXhap2QdkFl8CFBUPzf9Jc,7022
 codemie_sdk/services/datasource.py,sha256=gIP9D-sq4733x9f0mWL87bMI1pWYxIsv5eYXkAs0O6Q,7150
 codemie_sdk/services/integration.py,sha256=CrUfO-4_JwtCt8Qig3RP2uTi1uaTkedyTaPdkFY509g,5410
 codemie_sdk/services/llm.py,sha256=0-e4_7RvLHs2giCyoQ5U4KDTh6p5VXgPKNxnDP9ZDFU,1100
@@ -24,6 +25,6 @@ codemie_sdk/services/workflow_execution.py,sha256=aGoT3rdTmh5-doAsrmBBjLEuOfvL5a
 codemie_sdk/services/workflow_execution_state.py,sha256=tXoaa8yT09xgYEUNiHhVULe76TwGwVgZupMIUyyLxdo,2070
 codemie_sdk/utils/__init__.py,sha256=BXAJJfAzO89-kMYvWWo9wSNhSbGgF3vB1In9sePFhMM,109
 codemie_sdk/utils/http.py,sha256=YzE53y1r7PBUKcEQt-gGYduRY06I8xzqVRzlj9f_voA,10084
-codemie_sdk_python-0.1.73.dist-info/METADATA,sha256=3hLJJCC6C0AIgvYPh6B1xBS3ECfhQ0wnYKh6bVMZMYY,21852
-codemie_sdk_python-0.1.73.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-codemie_sdk_python-0.1.73.dist-info/RECORD,,
+codemie_sdk_python-0.1.75.dist-info/METADATA,sha256=EMn_rsOcCBy5bVfYHLQdXRSYT2XvvwbvePmRfvJytFg,21914
+codemie_sdk_python-0.1.75.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+codemie_sdk_python-0.1.75.dist-info/RECORD,,