pygeai 0.6.0b7__py3-none-any.whl → 0.6.0b11__py3-none-any.whl

pygeai/tests/snippets/evaluation/dataset/list_dataset_rows.py

@@ -0,0 +1,9 @@
+from pygeai.evaluation.dataset.clients import EvaluationDatasetClient
+
+client = EvaluationDatasetClient()
+
+dataset_id = "your-dataset-id-here"
+
+response = client.list_dataset_rows(dataset_id=dataset_id)
+
+print(f"Dataset rows: {response}")

pygeai/tests/snippets/evaluation/dataset/list_datasets.py

@@ -0,0 +1,6 @@
+from pygeai.evaluation.dataset.clients import EvaluationDatasetClient
+
+client = EvaluationDatasetClient()
+
+response = client.list_datasets()
+print(f"Datasets: {response}")

pygeai/tests/snippets/evaluation/dataset/list_expected_sources.py

@@ -0,0 +1,10 @@
+from pygeai.evaluation.dataset.clients import EvaluationDatasetClient
+
+client = EvaluationDatasetClient()
+
+dataset_id = "your-dataset-id-here"
+row_id = "your-row-id-here"
+
+response = client.list_expected_sources(dataset_id=dataset_id, dataset_row_id=row_id)
+
+print(f"Expected sources: {response}")

pygeai/tests/snippets/evaluation/dataset/list_filter_variables.py

@@ -0,0 +1,10 @@
+from pygeai.evaluation.dataset.clients import EvaluationDatasetClient
+
+client = EvaluationDatasetClient()
+
+dataset_id = "your-dataset-id-here"
+row_id = "your-row-id-here"
+
+response = client.list_filter_variables(dataset_id=dataset_id, dataset_row_id=row_id)
+
+print(f"Filter variables: {response}")

pygeai/tests/snippets/evaluation/dataset/update_dataset.py

@@ -0,0 +1,15 @@
+from pygeai.evaluation.dataset.clients import EvaluationDatasetClient
+
+client = EvaluationDatasetClient()
+
+dataset_id = "your-dataset-id-here"
+
+response = client.update_dataset(
+    dataset_id=dataset_id,
+    dataset_name="Updated Dataset Name",
+    dataset_description="Updated description",
+    dataset_type="E",
+    dataset_active=False
+)
+
+print(f"Updated dataset: {response}")

pygeai/tests/snippets/evaluation/dataset/update_dataset_row.py

@@ -0,0 +1,20 @@
+from pygeai.evaluation.dataset.clients import EvaluationDatasetClient
+
+client = EvaluationDatasetClient()
+
+dataset_id = "your-dataset-id-here"
+row_id = "your-row-id-here"
+
+updated_row = {
+    "dataSetRowInput": "Updated question: What is Machine Learning?",
+    "dataSetRowExpectedAnswer": "Updated answer: ML is a subset of AI",
+    "dataSetRowContextDocument": "Updated context about ML"
+}
+
+response = client.update_dataset_row(
+    dataset_id=dataset_id,
+    dataset_row_id=row_id,
+    row=updated_row
+)
+
+print(f"Updated row: {response}")

pygeai/tests/snippets/evaluation/dataset/update_expected_source.py

@@ -0,0 +1,18 @@
+from pygeai.evaluation.dataset.clients import EvaluationDatasetClient
+
+client = EvaluationDatasetClient()
+
+dataset_id = "your-dataset-id-here"
+row_id = "your-row-id-here"
+source_id = "your-expected-source-id-here"
+
+response = client.update_expected_source(
+    dataset_id=dataset_id,
+    dataset_row_id=row_id,
+    expected_source_id=source_id,
+    expected_source_name="Updated Documentation",
+    expected_source_value="Updated content from the source...",
+    expected_source_extension="txt"
+)
+
+print(f"Updated expected source: {response}")

pygeai/tests/snippets/evaluation/dataset/update_filter_variable.py

@@ -0,0 +1,19 @@
+from pygeai.evaluation.dataset.clients import EvaluationDatasetClient
+
+client = EvaluationDatasetClient()
+
+dataset_id = "your-dataset-id-here"
+row_id = "your-row-id-here"
+filter_var_id = "your-filter-variable-id-here"
+
+response = client.update_filter_variable(
+    dataset_id=dataset_id,
+    dataset_row_id=row_id,
+    filter_variable_id=filter_var_id,
+    metadata_type="V",
+    filter_variable_key="category",
+    filter_variable_value="technology",
+    filter_variable_operator="="
+)
+
+print(f"Updated filter variable: {response}")

pygeai/tests/snippets/evaluation/dataset/upload_dataset_rows_file.py

@@ -0,0 +1,10 @@
+from pygeai.evaluation.dataset.clients import EvaluationDatasetClient
+
+client = EvaluationDatasetClient()
+
+dataset_id = "your-dataset-id-here"
+file_path = "path/to/rows.json"
+
+response = client.upload_dataset_rows_file(dataset_id=dataset_id, file_path=file_path)
+
+print(f"Uploaded rows from file: {response}")

pygeai/tests/snippets/evaluation/plan/add_plan_system_metric.py

@@ -0,0 +1,13 @@
+from pygeai.evaluation.plan.clients import EvaluationPlanClient
+
+client = EvaluationPlanClient()
+
+evaluation_plan_id = "your-evaluation-plan-id"
+
+response = client.add_evaluation_plan_system_metric(
+    evaluation_plan_id=evaluation_plan_id,
+    system_metric_id="metric-id",
+    system_metric_weight=0.5
+)
+
+print(f"Added system metric: {response}")

pygeai/tests/snippets/evaluation/plan/complete_workflow_example.py

@@ -0,0 +1,136 @@
+"""
+Complete Evaluation Plan API Workflow Example
+
+This example demonstrates a full workflow using the Evaluation Plan API:
+1. Query available system metrics
+2. Create an evaluation plan
+3. Add system metrics
+4. Update the plan
+5. Execute the plan
+6. Cleanup
+"""
+
+from pygeai.evaluation.plan.clients import EvaluationPlanClient
+
+
+def main():
+    # Initialize client
+    client = EvaluationPlanClient()
+    
+    print("="*60)
+    print("Evaluation Plan API Complete Workflow Example")
+    print("="*60)
+    
+    # Step 1: List available system metrics
+    print("\n[1/7] Listing available system metrics...")
+    metrics = client.list_system_metrics()
+    print(f"✓ Available metrics: {metrics}")
+    
+    # Get details of a specific metric
+    if metrics and 'systemMetrics' in metrics and len(metrics['systemMetrics']) > 0:
+        first_metric_id = metrics['systemMetrics'][0].get('systemMetricId')
+        metric_details = client.get_system_metric(system_metric_id=first_metric_id)
+        print(f"✓ Sample metric details: {metric_details}")
+    
+    # Step 2: Create an evaluation plan
+    print("\n[2/7] Creating evaluation plan...")
+    plan = client.create_evaluation_plan(
+        name="AI Assistant Performance Test",
+        type="TextPromptAssistant",
+        assistant_id="your-assistant-id",
+        assistant_name="Test Assistant",
+        assistant_revision="1.0",
+        dataset_id="your-dataset-id",
+        system_metrics=[
+            {
+                "systemMetricId": "accuracy",
+                "systemMetricWeight": 0.6
+            },
+            {
+                "systemMetricId": "fluency",
+                "systemMetricWeight": 0.4
+            }
+        ]
+    )
+    plan_id = plan.get('evaluationPlanId')
+    print(f"✓ Created evaluation plan: {plan_id}")
+    
+    # Step 3: Get the plan details
+    print("\n[3/7] Retrieving plan details...")
+    plan_details = client.get_evaluation_plan(evaluation_plan_id=plan_id)
+    print(f"✓ Plan name: {plan_details.get('evaluationPlanName')}")
+    print(f"✓ Plan type: {plan_details.get('evaluationPlanType')}")
+    print(f"✓ Number of metrics: {len(plan_details.get('systemMetrics', []))}")
+    
+    # Step 4: List plan's system metrics
+    print("\n[4/7] Listing plan's system metrics...")
+    plan_metrics = client.list_evaluation_plan_system_metrics(evaluation_plan_id=plan_id)
+    print(f"✓ Plan metrics: {plan_metrics}")
+    
+    # Step 5: Add a new metric to the plan
+    print("\n[5/7] Adding new system metric to plan...")
+    new_metric = client.add_evaluation_plan_system_metric(
+        evaluation_plan_id=plan_id,
+        system_metric_id="relevance",
+        system_metric_weight=0.5
+    )
+    print(f"✓ Added metric: {new_metric}")
+    
+    # Step 6: Update a metric's weight
+    print("\n[6/7] Updating metric weight...")
+    updated_metric = client.update_evaluation_plan_system_metric(
+        evaluation_plan_id=plan_id,
+        system_metric_id="accuracy",
+        system_metric_weight=0.8
+    )
+    print(f"✓ Updated metric weight")
+    
+    # Get specific metric details
+    metric_detail = client.get_evaluation_plan_system_metric(
+        evaluation_plan_id=plan_id,
+        system_metric_id="accuracy"
+    )
+    print(f"✓ Metric details: {metric_detail}")
+    
+    # Step 7: Update the plan itself
+    print("\n[7/7] Updating evaluation plan...")
+    updated_plan = client.update_evaluation_plan(
+        evaluation_plan_id=plan_id,
+        name="Updated Performance Test",
+        assistant_revision="2.0"
+    )
+    print(f"✓ Updated plan")
+    
+    # Execute the evaluation plan
+    print("\n[EXECUTE] Running evaluation plan...")
+    execution = client.execute_evaluation_plan(evaluation_plan_id=plan_id)
+    print(f"✓ Execution started")
+    print(f"  Execution ID: {execution.get('executionId')}")
+    print(f"  Status: {execution.get('status')}")
+    
+    # Cleanup
+    print("\n[CLEANUP] Cleaning up...")
+    
+    # Delete metrics
+    client.delete_evaluation_plan_system_metric(
+        evaluation_plan_id=plan_id,
+        system_metric_id="relevance"
+    )
+    print(f"✓ Deleted added metric")
+    
+    # Delete the plan
+    client.delete_evaluation_plan(evaluation_plan_id=plan_id)
+    print(f"✓ Deleted evaluation plan")
+    
+    print("\n" + "="*60)
+    print("Workflow completed successfully!")
+    print("="*60)
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except Exception as e:
+        print(f"\n❌ Error occurred: {e}")
+        import traceback
+        traceback.print_exc()

pygeai/tests/snippets/evaluation/plan/create_evaluation_plan.py

@@ -0,0 +1,24 @@
+from pygeai.evaluation.plan.clients import EvaluationPlanClient
+
+client = EvaluationPlanClient()
+
+response = client.create_evaluation_plan(
+    name="My Evaluation Plan",
+    type="TextPromptAssistant",
+    assistant_id="your-assistant-id",
+    assistant_name="My Assistant",
+    assistant_revision="1.0",
+    dataset_id="your-dataset-id",
+    system_metrics=[
+        {
+            "systemMetricId": "metric-1",
+            "systemMetricWeight": 0.6
+        },
+        {
+            "systemMetricId": "metric-2",
+            "systemMetricWeight": 0.4
+        }
+    ]
+)
+
+print(f"Created evaluation plan: {response}")

pygeai/tests/snippets/evaluation/plan/create_rag_evaluation_plan.py

@@ -0,0 +1,22 @@
+from pygeai.evaluation.plan.clients import EvaluationPlanClient
+
+client = EvaluationPlanClient()
+
+response = client.create_evaluation_plan(
+    name="RAG Assistant Evaluation",
+    type="RAG Assistant",
+    profile_name="My RAG Profile",
+    dataset_id="your-dataset-id",
+    system_metrics=[
+        {
+            "systemMetricId": "accuracy",
+            "systemMetricWeight": 0.7
+        },
+        {
+            "systemMetricId": "fluency",
+            "systemMetricWeight": 0.3
+        }
+    ]
+)
+
+print(f"Created RAG evaluation plan: {response}")

pygeai/tests/snippets/evaluation/plan/delete_evaluation_plan.py

@@ -0,0 +1,9 @@
+from pygeai.evaluation.plan.clients import EvaluationPlanClient
+
+client = EvaluationPlanClient()
+
+evaluation_plan_id = "your-evaluation-plan-id"
+
+response = client.delete_evaluation_plan(evaluation_plan_id=evaluation_plan_id)
+
+print(f"Deleted evaluation plan: {response}")

pygeai/tests/snippets/evaluation/plan/delete_plan_system_metric.py

@@ -0,0 +1,13 @@
+from pygeai.evaluation.plan.clients import EvaluationPlanClient
+
+client = EvaluationPlanClient()
+
+evaluation_plan_id = "your-evaluation-plan-id"
+system_metric_id = "your-metric-id"
+
+response = client.delete_evaluation_plan_system_metric(
+    evaluation_plan_id=evaluation_plan_id,
+    system_metric_id=system_metric_id
+)
+
+print(f"Deleted system metric: {response}")

pygeai/tests/snippets/evaluation/plan/execute_evaluation_plan.py

@@ -0,0 +1,11 @@
+from pygeai.evaluation.plan.clients import EvaluationPlanClient
+
+client = EvaluationPlanClient()
+
+evaluation_plan_id = "your-evaluation-plan-id"
+
+response = client.execute_evaluation_plan(evaluation_plan_id=evaluation_plan_id)
+
+print(f"Execution started: {response}")
+print(f"Execution ID: {response.get('executionId')}")
+print(f"Status: {response.get('status')}")

pygeai/tests/snippets/evaluation/plan/get_evaluation_plan.py

@@ -0,0 +1,9 @@
+from pygeai.evaluation.plan.clients import EvaluationPlanClient
+
+client = EvaluationPlanClient()
+
+evaluation_plan_id = "your-evaluation-plan-id"
+
+response = client.get_evaluation_plan(evaluation_plan_id=evaluation_plan_id)
+
+print(f"Evaluation plan: {response}")

pygeai/tests/snippets/evaluation/plan/get_plan_system_metric.py

@@ -0,0 +1,13 @@
+from pygeai.evaluation.plan.clients import EvaluationPlanClient
+
+client = EvaluationPlanClient()
+
+evaluation_plan_id = "your-evaluation-plan-id"
+system_metric_id = "your-metric-id"
+
+response = client.get_evaluation_plan_system_metric(
+    evaluation_plan_id=evaluation_plan_id,
+    system_metric_id=system_metric_id
+)
+
+print(f"Plan system metric: {response}")

pygeai/tests/snippets/evaluation/plan/get_system_metric.py

@@ -0,0 +1,9 @@
+from pygeai.evaluation.plan.clients import EvaluationPlanClient
+
+client = EvaluationPlanClient()
+
+system_metric_id = "your-metric-id"
+
+response = client.get_system_metric(system_metric_id=system_metric_id)
+
+print(f"System metric: {response}")

pygeai/tests/snippets/evaluation/plan/list_evaluation_plans.py

@@ -0,0 +1,7 @@
+from pygeai.evaluation.plan.clients import EvaluationPlanClient
+
+client = EvaluationPlanClient()
+
+response = client.list_evaluation_plans()
+
+print(f"Evaluation plans: {response}")

pygeai/tests/snippets/evaluation/plan/list_plan_system_metrics.py

@@ -0,0 +1,9 @@
+from pygeai.evaluation.plan.clients import EvaluationPlanClient
+
+client = EvaluationPlanClient()
+
+evaluation_plan_id = "your-evaluation-plan-id"
+
+response = client.list_evaluation_plan_system_metrics(evaluation_plan_id=evaluation_plan_id)
+
+print(f"Plan system metrics: {response}")

pygeai/tests/snippets/evaluation/plan/list_system_metrics.py

@@ -0,0 +1,7 @@
+from pygeai.evaluation.plan.clients import EvaluationPlanClient
+
+client = EvaluationPlanClient()
+
+response = client.list_system_metrics()
+
+print(f"Available system metrics: {response}")

pygeai/tests/snippets/evaluation/plan/update_evaluation_plan.py

@@ -0,0 +1,22 @@
+from pygeai.evaluation.plan.clients import EvaluationPlanClient
+
+client = EvaluationPlanClient()
+
+evaluation_plan_id = "your-evaluation-plan-id"
+
+response = client.update_evaluation_plan(
+    evaluation_plan_id=evaluation_plan_id,
+    name="Updated Plan Name",
+    system_metrics=[
+        {
+            "systemMetricId": "metric-1",
+            "systemMetricWeight": 0.8
+        },
+        {
+            "systemMetricId": "metric-2",
+            "systemMetricWeight": 0.2
+        }
+    ]
+)
+
+print(f"Updated evaluation plan: {response}")

pygeai/tests/snippets/evaluation/plan/update_plan_system_metric.py

@@ -0,0 +1,14 @@
+from pygeai.evaluation.plan.clients import EvaluationPlanClient
+
+client = EvaluationPlanClient()
+
+evaluation_plan_id = "your-evaluation-plan-id"
+system_metric_id = "your-metric-id"
+
+response = client.update_evaluation_plan_system_metric(
+    evaluation_plan_id=evaluation_plan_id,
+    system_metric_id=system_metric_id,
+    system_metric_weight=0.9
+)
+
+print(f"Updated system metric: {response}")

pygeai/tests/snippets/evaluation/result/complete_workflow_example.py

@@ -0,0 +1,150 @@
+"""
+Complete Evaluation Result API Workflow Example
+
+This example demonstrates how to retrieve and analyze evaluation results:
+1. List all results for a plan
+2. Get detailed results for each
+3. Analyze performance metrics
+4. Extract insights
+
+Note: This is a read-only API. Results are created by executing evaluation plans.
+"""
+
+from pygeai.evaluation.result.clients import EvaluationResultClient
+
+
+def main():
+    # Initialize client
+    client = EvaluationResultClient()
+    
+    print("="*70)
+    print("Evaluation Result API Complete Workflow Example")
+    print("="*70)
+    
+    # Step 1: List all results for an evaluation plan
+    print("\n[1/4] Listing evaluation results for a plan...")
+    evaluation_plan_id = "your-evaluation-plan-id"
+    
+    results = client.list_evaluation_results(evaluation_plan_id=evaluation_plan_id)
+    
+    if isinstance(results, list):
+        print(f"✓ Found {len(results)} evaluation results")
+    else:
+        print(f"✓ Results: {results}")
+        results = []
+    
+    # Step 2: Display summary of all results
+    print("\n[2/4] Summary of all results:")
+    print("-" * 70)
+    
+    total_cost = 0
+    total_duration = 0
+    status_counts = {}
+    
+    for i, result in enumerate(results, 1):
+        result_id = result.get('evaluationResultId', 'Unknown')
+        status = result.get('evaluationResultStatus', 'Unknown')
+        cost = result.get('evaluationResultCost', 0)
+        duration = result.get('evaluationResultDuration', 0)
+        
+        print(f"\n{i}. Result ID: {result_id}")
+        print(f"   Status: {status}")
+        print(f"   Cost: ${cost}")
+        print(f"   Duration: {duration}ms")
+        print(f"   Model: {result.get('evaluationResultModelName', 'N/A')}")
+        print(f"   Provider: {result.get('evaluationResultProviderName', 'N/A')}")
+        
+        # Aggregate metrics
+        total_cost += cost
+        total_duration += duration
+        status_counts[status] = status_counts.get(status, 0) + 1
+    
+    # Step 3: Get detailed results for the first result
+    if results:
+        print("\n[3/4] Getting detailed results for first evaluation...")
+        first_result_id = results[0].get('evaluationResultId')
+        
+        detailed_result = client.get_evaluation_result(
+            evaluation_result_id=first_result_id
+        )
+        
+        print(f"✓ Retrieved detailed result: {detailed_result.get('evaluationResultId')}")
+        
+        # Display row-level details
+        rows = detailed_result.get('rows', [])
+        print(f"\n  Row-level results: {len(rows)} rows")
+        
+        successful_rows = 0
+        failed_rows = 0
+        row_costs = []
+        
+        for row in rows:
+            row_status = row.get('evaluationResultRowStatus', 'Unknown')
+            row_cost = row.get('evaluationResultRowCost', 0)
+            
+            if row_status == 'completed' or row_status == 'success':
+                successful_rows += 1
+            else:
+                failed_rows += 1
+            
+            row_costs.append(row_cost)
+        
+        print(f"  Successful rows: {successful_rows}")
+        print(f"  Failed rows: {failed_rows}")
+        
+        if row_costs:
+            avg_row_cost = sum(row_costs) / len(row_costs)
+            print(f"  Average row cost: ${avg_row_cost:.4f}")
+        
+        # Show sample row
+        if rows:
+            print(f"\n  Sample Row:")
+            sample_row = rows[0]
+            print(f"    Dataset Row ID: {sample_row.get('dataSetRowId')}")
+            print(f"    Status: {sample_row.get('evaluationResultRowStatus')}")
+            print(f"    Cost: ${sample_row.get('evaluationResultRowCost')}")
+            print(f"    Start: {sample_row.get('evaluationResultRowStartDate')}")
+            print(f"    End: {sample_row.get('evaluationResultRowEndDate')}")
+            
+            output = sample_row.get('evaluationResultRowOutput', '')
+            if output:
+                print(f"    Output (first 200 chars):")
+                print(f"      {output[:200]}...")
+    
+    # Step 4: Display aggregated analytics
+    print("\n[4/4] Aggregated Analytics:")
+    print("-" * 70)
+    print(f"Total Evaluations: {len(results)}")
+    print(f"Total Cost: ${total_cost:.2f}")
+    print(f"Total Duration: {total_duration}ms ({total_duration/1000:.2f}s)")
+    
+    if results:
+        print(f"Average Cost per Evaluation: ${total_cost/len(results):.4f}")
+        print(f"Average Duration per Evaluation: {total_duration/len(results):.0f}ms")
+    
+    print(f"\nStatus Distribution:")
+    for status, count in status_counts.items():
+        percentage = (count / len(results) * 100) if results else 0
+        print(f"  {status}: {count} ({percentage:.1f}%)")
+    
+    print("\n" + "="*70)
+    print("Workflow completed successfully!")
+    print("="*70)
+    
+    # Important note about field names
+    print("\n⚠️  IMPORTANT NOTE:")
+    print("The API responses contain typos in some field names:")
+    print("  - evaluationResultAssitantRevision (missing 's' in Assistant)")
+    print("  - evaluationResultChunckCount (should be Chunk)")
+    print("  - evaluationResultChunckSize (should be Chunk)")
+    print("  - evaluationResultaMaxTokens (lowercase 'a')")
+    print("\nThese are API-level typos, not errors in our code.")
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except Exception as e:
+        print(f"\n❌ Error occurred: {e}")
+        import traceback
+        traceback.print_exc()

pygeai/tests/snippets/evaluation/result/get_evaluation_result.py

@@ -0,0 +1,26 @@
+from pygeai.evaluation.result.clients import EvaluationResultClient
+
+client = EvaluationResultClient()
+
+evaluation_result_id = "your-evaluation-result-id"
+
+result = client.get_evaluation_result(evaluation_result_id=evaluation_result_id)
+
+print(f"Evaluation Result: {result.get('evaluationResultId')}")
+print(f"Status: {result.get('evaluationResultStatus')}")
+print(f"Plan ID: {result.get('evaluationPlanId')}")
+print(f"Dataset ID: {result.get('dataSetId')}")
+print(f"Cost: ${result.get('evaluationResultCost')}")
+print(f"Duration: {result.get('evaluationResultDuration')}ms")
+print(f"Model: {result.get('evaluationResultModelName')}")
+print(f"Provider: {result.get('evaluationResultProviderName')}")
+
+# Row-level details
+rows = result.get('rows', [])
+print(f"\nRow-level results: {len(rows)} rows")
+
+for row in rows:
+    print(f"\n  Row ID: {row.get('dataSetRowId')}")
+    print(f"    Status: {row.get('evaluationResultRowStatus')}")
+    print(f"    Cost: ${row.get('evaluationResultRowCost')}")
+    print(f"    Output: {row.get('evaluationResultRowOutput')[:100]}...")