eval-studio-client 1.1.0a5__py3-none-any.whl → 1.1.0a7__py3-none-any.whl

eval_studio_client/gen/openapiv2/eval_studio.swagger.json

@@ -20,9 +20,6 @@
     {
       "name": "EvaluatorService"
     },
-    {
-      "name": "GeneratedQuestionsValidationService"
-    },
     {
       "name": "HumanCalibrationService"
     },
@@ -245,6 +242,13 @@
             "required": false,
             "type": "integer",
             "format": "int32"
+          },
+          {
+            "name": "filter",
+            "description": "Optional. If specified, only leaderboards matching the filter will be returned. Attempts to\nimplement AIP-160 (https://aip.dev/160), although not all fields, operators and features are\nsupported.\n\nSupported fields:\n- model\n  - only '=' operator is supported, e.g. \"model = \\\"models/\u003cUUID\u003e\\\"\"",
+            "in": "query",
+            "required": false,
+            "type": "string"
           }
         ],
         "tags": [
@@ -1914,6 +1918,13 @@
           }
         },
         "parameters": [
+          {
+            "name": "filter",
+            "description": "Optional. If specified, only leaderboards matching the filter will be returned. Attempts to\nimplement AIP-160 (https://aip.dev/160), although not all fields, operators and features are\nsupported.\n\nSupported fields:\n- type\n  - only '=' operator is supported",
+            "in": "query",
+            "required": false,
+            "type": "string"
+          },
           {
             "name": "orderBy",
             "description": "If specified, the returned tests will be ordered by the specified field. Attempts to implement\nAIP-130 (https://google.aip.dev/132#ordering), although not all features are supported yet.\n\nSupported fields:\n- create_time\n- update_time",
@@ -2117,6 +2128,13 @@
             "required": false,
             "type": "integer",
             "format": "int32"
+          },
+          {
+            "name": "filter",
+            "description": "Optional. If specified, only leaderboards matching the filter will be returned. Attempts to\nimplement AIP-160 (https://aip.dev/160), although not all fields, operators and features are\nsupported.\n\nSupported fields:\n- type\n  - only '=' operator is supported",
+            "in": "query",
+            "required": false,
+            "type": "string"
           }
         ],
         "tags": [
@@ -2416,6 +2434,10 @@
                   "type": "boolean",
                   "description": "Output only. Whether the Dashboard is a demo resource or not. Demo resources are read only.",
                   "readOnly": true
+                },
+                "type": {
+                  "$ref": "#/definitions/v1DashboardType",
+                  "description": "Optional. Type of the dashboard. Defaults to DASHBOARD_TYPE_STANDALONE for\nCreateDashboard and CreateDashboardWithoutCache. Defaults to DASHBOARD_TYPE_STANDALO for\nBatchCreateDashboards."
                 }
               },
               "description": "The Dashboard's `name` field is used to identify the Dashboard to update. Format:\ndashboards/{dashboard}",
@@ -3691,7 +3713,7 @@
                   "description": "The list of document URLs. The document URL might be a managed document URL or a\npublic URL."
                 },
                 "chunks": {
-                  "$ref": "#/definitions/v1RepeatedString",
+                  "$ref": "#/definitions/v1RepeatedContext",
                   "title": "Chunks of text used instead of documents"
                 },
                 "generators": {
@@ -3838,7 +3860,8 @@
                 "chunks": {
                   "type": "array",
                   "items": {
-                    "type": "string"
+                    "type": "object",
+                    "$ref": "#/definitions/v1Context"
                   },
                   "description": "Optional. The list of chunks to use for generation. If set, the Documents assigned to the Test\nand h2ogpte_collection_id are ignored."
                 }
@@ -3851,6 +3874,38 @@
         ]
       }
     },
+    "/v1/{name}:getGuardrailsConfiguration": {
+      "get": {
+        "operationId": "WorkflowService_GetGuardrailsConfiguration",
+        "responses": {
+          "200": {
+            "description": "A successful response.",
+            "schema": {
+              "$ref": "#/definitions/v1GetGuardrailsConfigurationResponse"
+            }
+          },
+          "default": {
+            "description": "An unexpected error response.",
+            "schema": {
+              "$ref": "#/definitions/rpcStatus"
+            }
+          }
+        },
+        "parameters": [
+          {
+            "name": "name",
+            "description": "Required. The name of the Workflow to retrieve guardrails configuration for. The Human\nCalibration node must be completed, otherwise an error is returned.",
+            "in": "path",
+            "required": true,
+            "type": "string",
+            "pattern": "workflows/[^/]+"
+          }
+        ],
+        "tags": [
+          "WorkflowService"
+        ]
+      }
+    },
     "/v1/{name}:getPrerequisites": {
       "get": {
         "summary": "GetWorkflowNodePrerequisites retrieves the WorkflowNodes and WorkflowEdges that are the\nprerequisites of the specified WorkflowNode. The list might be empty. Large data might be\nstored in storage and not returned in the response. It is client's responsibility to retrieve\nthe data from storage using the content handlers. It is intended to be used by the Eval Studio\nWorkers.",
@@ -4060,6 +4115,39 @@
         ]
       }
     },
+    "/v1/{name}:nodeDependencies": {
+      "get": {
+        "summary": "ListWorkflowDependencies lists workflow nodes and map of the node dependencies.",
+        "operationId": "WorkflowService_ListWorkflowDependencies",
+        "responses": {
+          "200": {
+            "description": "A successful response.",
+            "schema": {
+              "$ref": "#/definitions/v1ListWorkflowDependenciesResponse"
+            }
+          },
+          "default": {
+            "description": "An unexpected error response.",
+            "schema": {
+              "$ref": "#/definitions/rpcStatus"
+            }
+          }
+        },
+        "parameters": [
+          {
+            "name": "name",
+            "description": "Required. The name of the Workflow to retrieve dependencies for.",
+            "in": "path",
+            "required": true,
+            "type": "string",
+            "pattern": "workflows/[^/]+"
+          }
+        ],
+        "tags": [
+          "WorkflowService"
+        ]
+      }
+    },
     "/v1/{name}:perturb": {
       "post": {
         "operationId": "TestService_PerturbTest",
@@ -4248,60 +4336,6 @@
         ]
       }
     },
-    "/v1/{name}:validateGeneratedQuestions": {
-      "post": {
-        "operationId": "GeneratedQuestionsValidationService_ValidateGeneratedQuestions",
-        "responses": {
-          "200": {
-            "description": "A successful response.",
-            "schema": {
-              "$ref": "#/definitions/v1Operation"
-            }
-          },
-          "default": {
-            "description": "An unexpected error response.",
-            "schema": {
-              "$ref": "#/definitions/rpcStatus"
-            }
-          }
-        },
-        "parameters": [
-          {
-            "name": "name",
-            "description": "Required. The Test for which to generate TestCases.",
-            "in": "path",
-            "required": true,
-            "type": "string",
-            "pattern": "tests/[^/]+"
-          },
-          {
-            "name": "body",
-            "in": "body",
-            "required": true,
-            "schema": {
-              "type": "object",
-              "properties": {
-                "operation": {
-                  "type": "string",
-                  "description": "Required. The Operation processing this question validation process."
-                },
-                "testCases": {
-                  "type": "array",
-                  "items": {
-                    "type": "object",
-                    "$ref": "#/definitions/v1GeneratedTestCase"
-                  },
-                  "description": "Required. Generated Test Cases, i.e., Test cases with context that was used for their generation."
-                }
-              }
-            }
-          }
-        ],
-        "tags": [
-          "GeneratedQuestionsValidationService"
-        ]
-      }
-    },
     "/v1/{node.name}": {
       "patch": {
         "summary": "UpdateWorkflowNode updates a WorkflowNode. The update_mask is used to specify the fields to be\nupdated.",
@@ -5048,6 +5082,10 @@
                   "type": "boolean",
                   "description": "Output only. Whether the Test is a demo resource or not. Demo resources are read only.",
                   "readOnly": true
+                },
+                "type": {
+                  "$ref": "#/definitions/v1TestType",
+                  "description": "Optional. Type of the test. Defaults to TEST_TYPE_STANDALONE for\nCreateTest and CreateTestWithoutCache. Defaults to TEST_TYPE_STANDALONE for\nBatchCreateTests."
                 }
               },
               "title": "Required. The Test to update."
@@ -6023,21 +6061,42 @@
       "properties": {
         "collectionId": {
           "type": "string",
-          "description": "Collection Id."
+          "description": "Optional. Collection Id."
         },
         "chunkId": {
           "type": "integer",
           "format": "int32",
-          "description": "Chunk Id."
+          "description": "Required. Chunk Id."
         },
         "score": {
           "type": "number",
           "format": "float",
-          "description": "Chunk score."
+          "description": "Optional. Chunk score."
         },
         "content": {
           "type": "string",
-          "description": "Content."
+          "description": "Required. Content."
+        },
+        "documentId": {
+          "type": "string",
+          "description": "Optional. Document Id."
+        },
+        "documentName": {
+          "type": "string",
+          "description": "Optional. Document name."
+        },
+        "pages": {
+          "type": "string",
+          "description": "Optional. Pages."
+        },
+        "topic": {
+          "type": "string",
+          "description": "Required. Topic."
+        },
+        "topicId": {
+          "type": "integer",
+          "format": "int32",
+          "description": "Required. Topic Id."
         }
       }
     },
@@ -6295,6 +6354,10 @@
           "type": "boolean",
           "description": "Output only. Whether the Dashboard is a demo resource or not. Demo resources are read only.",
           "readOnly": true
+        },
+        "type": {
+          "$ref": "#/definitions/v1DashboardType",
+          "description": "Optional. Type of the dashboard. Defaults to DASHBOARD_TYPE_STANDALONE for\nCreateDashboard and CreateDashboardWithoutCache. Defaults to DASHBOARD_TYPE_STANDALO for\nBatchCreateDashboards."
         }
       }
     },
@@ -6309,6 +6372,16 @@
       "default": "DASHBOARD_STATUS_UNSPECIFIED",
       "description": " - DASHBOARD_STATUS_UNSPECIFIED: Unspecified status.\n - DASHBOARD_STATUS_PROCESSING: Dashboard is being processed. At least one of the leaderboard has a processing status (but no failed status).\n - DASHBOARD_STATUS_COMPLETED: Dashboard is completed successfully. All of the leaderboards are completed.\n - DASHBOARD_STATUS_FAILED: Dashboard failed. At least one of the leaderboard failed."
     },
+    "v1DashboardType": {
+      "type": "string",
+      "enum": [
+        "DASHBOARD_TYPE_UNSPECIFIED",
+        "DASHBOARD_TYPE_STANDALONE",
+        "DASHBOARD_TYPE_WORKFLOW"
+      ],
+      "default": "DASHBOARD_TYPE_UNSPECIFIED",
+      "description": " - DASHBOARD_TYPE_UNSPECIFIED: Unspecified type.\n - DASHBOARD_TYPE_STANDALONE: Standalone dashboard.\n - DASHBOARD_TYPE_WORKFLOW: Dashboard is part of a dashboard group."
+    },
     "v1DeleteDashboardResponse": {
       "type": "object",
       "properties": {
@@ -6399,6 +6472,23 @@
         }
       }
     },
+    "v1DependencyList": {
+      "type": "object",
+      "properties": {
+        "node": {
+          "type": "string",
+          "description": "The name of the Workflow Node to retrieve dependencies for."
+        },
+        "dependencies": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "$ref": "#/definitions/v1WorkflowDependency"
+          },
+          "description": "The names of the Workflow Node dependencies."
+        }
+      }
+    },
     "v1Document": {
       "type": "object",
       "properties": {
@@ -6607,6 +6697,17 @@
           "type": "string",
           "description": "Output only. Tagline is a short (single-line) and high-level description of the evaluator.",
           "readOnly": true
+        },
+        "primaryMetric": {
+          "type": "string",
+          "description": "Output only. Optional. The name of the primary metric.",
+          "readOnly": true
+        },
+        "primaryMetricDefaultThreshold": {
+          "type": "number",
+          "format": "float",
+          "description": "Output only. Optional. Default threshold of the primary metric. Value must be ignored if\nprimary_metric is invalid.",
+          "readOnly": true
         }
       }
     },
@@ -6747,23 +6848,6 @@
         }
       }
     },
-    "v1GeneratedTestCase": {
-      "type": "object",
-      "properties": {
-        "testCase": {
-          "$ref": "#/definitions/v1TestCase",
-          "description": "Test Case."
-        },
-        "context": {
-          "type": "array",
-          "items": {
-            "type": "object",
-            "$ref": "#/definitions/v1Context"
-          },
-          "description": "Context used for test_case generation."
-        }
-      }
-    },
     "v1GetDashboardResponse": {
       "type": "object",
       "properties": {
@@ -6791,6 +6875,15 @@
         }
       }
     },
+    "v1GetGuardrailsConfigurationResponse": {
+      "type": "object",
+      "properties": {
+        "guardrailsConfigurationJson": {
+          "type": "string",
+          "description": "The guardrails configuration for the Workflow requested in JSON format. This configuration is\nusable by the guardrails client as is."
+        }
+      }
+    },
     "v1GetInfoResponse": {
       "type": "object",
       "properties": {
@@ -7671,20 +7764,10 @@
           "description": "Output only. List of documents.",
           "readOnly": true
         },
-        "modelCfg": {
-          "type": "object",
-          "description": "Output only. Model configuration.",
-          "readOnly": true
-        },
         "key": {
           "type": "string",
           "description": "Output only. Model key.",
           "readOnly": true
-        },
-        "llmModelMeta": {
-          "type": "object",
-          "description": "Output only. LLM model metadata - recursive dictionaries with an additional info like performance stats.",
-          "readOnly": true
         }
       },
       "description": "Model represents the evaluated model whose outputs were evaluated to create the results."
@@ -8134,6 +8217,27 @@
         }
       }
     },
+    "v1ListWorkflowDependenciesResponse": {
+      "type": "object",
+      "properties": {
+        "nodes": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "$ref": "#/definitions/v1WorkflowNode"
+          },
+          "description": "The list of the WorkflowNodes related to requested workflow."
+        },
+        "dependencies": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "$ref": "#/definitions/v1DependencyList"
+          },
+          "description": "The dependency map for the workflow."
+        }
+      }
+    },
     "v1ListWorkflowsResponse": {
       "type": "object",
       "properties": {
@@ -8612,6 +8716,18 @@
         }
       }
     },
+    "v1RepeatedContext": {
+      "type": "object",
+      "properties": {
+        "content": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "$ref": "#/definitions/v1Context"
+          }
+        }
+      }
+    },
     "v1RepeatedString": {
       "type": "object",
       "properties": {
@@ -8699,6 +8815,10 @@
           "type": "boolean",
           "description": "Output only. Whether the Test is a demo resource or not. Demo resources are read only.",
           "readOnly": true
+        },
+        "type": {
+          "$ref": "#/definitions/v1TestType",
+          "description": "Optional. Type of the test. Defaults to TEST_TYPE_STANDALONE for\nCreateTest and CreateTestWithoutCache. Defaults to TEST_TYPE_STANDALONE for\nBatchCreateTests."
         }
       }
     },
@@ -9001,6 +9121,16 @@
       ],
       "default": "TEST_SUITE_EVALUATES_UNSPECIFIED"
     },
+    "v1TestType": {
+      "type": "string",
+      "enum": [
+        "TEST_TYPE_UNSPECIFIED",
+        "TEST_TYPE_STANDALONE",
+        "TEST_TYPE_WORKFLOW"
+      ],
+      "default": "TEST_TYPE_UNSPECIFIED",
+      "description": " - TEST_TYPE_UNSPECIFIED: Unspecified type.\n - TEST_TYPE_STANDALONE: Standalone test.\n - TEST_TYPE_WORKFLOW: Test is part of a dashboard."
+    },
     "v1UpdateDashboardResponse": {
       "type": "object",
       "properties": {
@@ -9212,6 +9342,19 @@
       },
       "description": "Workflow represents a workflow in Eval Studio. It consists of WorkflowNodes and WorkflowEdges."
     },
+    "v1WorkflowDependency": {
+      "type": "object",
+      "properties": {
+        "name": {
+          "type": "string",
+          "description": "Required. The name of the Workflow Dependency node."
+        },
+        "optional": {
+          "type": "boolean",
+          "description": "Optional. Whether the dependency is optional."
+        }
+      }
+    },
     "v1WorkflowEdge": {
       "type": "object",
       "properties": {
@@ -9276,10 +9419,11 @@
       "type": "string",
       "enum": [
         "WORKFLOW_EDGE_TYPE_UNSPECIFIED",
-        "WORKFLOW_EDGE_TYPE_SIMPLE"
+        "WORKFLOW_EDGE_TYPE_DEPENDENCY",
+        "WORKFLOW_EDGE_TYPE_OPTIONAL_DEPENDENCY"
       ],
       "default": "WORKFLOW_EDGE_TYPE_UNSPECIFIED",
-      "description": "WorkflowEdgeType represents the type of a WorkflowEdge.\n\n - WORKFLOW_EDGE_TYPE_UNSPECIFIED: Unspecified type.\n - WORKFLOW_EDGE_TYPE_SIMPLE: Simple edge without any conditions."
+      "description": "WorkflowEdgeType represents the type of a WorkflowEdge.\n\n - WORKFLOW_EDGE_TYPE_UNSPECIFIED: Unspecified type.\n - WORKFLOW_EDGE_TYPE_DEPENDENCY: A required dependency type of edge.\nThis edge type defines that the \"to\" node depends on the outputs of the \"from\" node.\n - WORKFLOW_EDGE_TYPE_OPTIONAL_DEPENDENCY: Optional dependency edge, which indicates that the outputs of the \"from\" node can be used in case they exist,\nbut the \"to\" node does not rely on them."
     },
     "v1WorkflowNode": {
       "type": "object",
@@ -9430,6 +9574,11 @@
         "mimeType": {
           "type": "string",
           "description": "Optional MIME type of the WorkflowNodeArtifact."
+        },
+        "type": {
+          "type": "string",
+          "description": "Output only. Immutable. Type of the WorkflowNodeArtifact.",
+          "readOnly": true
         }
       },
       "description": "WorkflowNodeArtifact represents an artifact produced/consumed by a WorkflowNode."

{eval_studio_client-1.1.0a5.dist-info → eval_studio_client-1.1.0a7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-studio-client
-Version: 1.1.0a5
+Version: 1.1.0a7
 Project-URL: Source, https://github.com/h2oai/eval-studio/tree/main/client-py/src/
 Project-URL: Issues, https://github.com/h2oai/eval-studio/issues
 Author-email: "H2O.ai" <support@h2o.ai>