ibm-watsonx-gov 1.3.3__cp313-cp313-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ibm_watsonx_gov/__init__.py +8 -0
- ibm_watsonx_gov/agent_catalog/__init__.py +8 -0
- ibm_watsonx_gov/agent_catalog/clients/__init__.py +14 -0
- ibm_watsonx_gov/agent_catalog/clients/ai_agent_client.py +333 -0
- ibm_watsonx_gov/agent_catalog/core/__init__.py +8 -0
- ibm_watsonx_gov/agent_catalog/core/agent_loader.py +202 -0
- ibm_watsonx_gov/agent_catalog/core/agents.py +134 -0
- ibm_watsonx_gov/agent_catalog/entities/__init__.py +8 -0
- ibm_watsonx_gov/agent_catalog/entities/ai_agent.py +599 -0
- ibm_watsonx_gov/agent_catalog/utils/__init__.py +8 -0
- ibm_watsonx_gov/agent_catalog/utils/constants.py +36 -0
- ibm_watsonx_gov/agent_catalog/utils/notebook_utils.py +70 -0
- ibm_watsonx_gov/ai_experiments/__init__.py +8 -0
- ibm_watsonx_gov/ai_experiments/ai_experiments_client.py +980 -0
- ibm_watsonx_gov/ai_experiments/utils/__init__.py +8 -0
- ibm_watsonx_gov/ai_experiments/utils/ai_experiment_utils.py +139 -0
- ibm_watsonx_gov/clients/__init__.py +0 -0
- ibm_watsonx_gov/clients/api_client.py +99 -0
- ibm_watsonx_gov/clients/segment_client.py +46 -0
- ibm_watsonx_gov/clients/usage_client.cp313-win_amd64.pyd +0 -0
- ibm_watsonx_gov/clients/wx_ai_client.py +87 -0
- ibm_watsonx_gov/config/__init__.py +14 -0
- ibm_watsonx_gov/config/agentic_ai_configuration.py +225 -0
- ibm_watsonx_gov/config/gen_ai_configuration.py +129 -0
- ibm_watsonx_gov/config/model_risk_configuration.py +173 -0
- ibm_watsonx_gov/config/predictive_ai_configuration.py +20 -0
- ibm_watsonx_gov/entities/__init__.py +8 -0
- ibm_watsonx_gov/entities/agentic_app.py +209 -0
- ibm_watsonx_gov/entities/agentic_evaluation_result.py +185 -0
- ibm_watsonx_gov/entities/ai_evaluation.py +290 -0
- ibm_watsonx_gov/entities/ai_experiment.py +419 -0
- ibm_watsonx_gov/entities/base_classes.py +134 -0
- ibm_watsonx_gov/entities/container.py +54 -0
- ibm_watsonx_gov/entities/credentials.py +633 -0
- ibm_watsonx_gov/entities/criteria.py +508 -0
- ibm_watsonx_gov/entities/enums.py +274 -0
- ibm_watsonx_gov/entities/evaluation_result.py +444 -0
- ibm_watsonx_gov/entities/foundation_model.py +490 -0
- ibm_watsonx_gov/entities/llm_judge.py +44 -0
- ibm_watsonx_gov/entities/locale.py +17 -0
- ibm_watsonx_gov/entities/mapping.py +49 -0
- ibm_watsonx_gov/entities/metric.py +211 -0
- ibm_watsonx_gov/entities/metric_threshold.py +36 -0
- ibm_watsonx_gov/entities/model_provider.py +329 -0
- ibm_watsonx_gov/entities/model_risk_result.py +43 -0
- ibm_watsonx_gov/entities/monitor.py +71 -0
- ibm_watsonx_gov/entities/prompt_setup.py +40 -0
- ibm_watsonx_gov/entities/state.py +22 -0
- ibm_watsonx_gov/entities/utils.py +99 -0
- ibm_watsonx_gov/evaluators/__init__.py +26 -0
- ibm_watsonx_gov/evaluators/agentic_evaluator.py +2725 -0
- ibm_watsonx_gov/evaluators/agentic_traces_evaluator.py +115 -0
- ibm_watsonx_gov/evaluators/base_evaluator.py +22 -0
- ibm_watsonx_gov/evaluators/impl/__init__.py +0 -0
- ibm_watsonx_gov/evaluators/impl/evaluate_metrics_impl.cp313-win_amd64.pyd +0 -0
- ibm_watsonx_gov/evaluators/impl/evaluate_model_risk_impl.cp313-win_amd64.pyd +0 -0
- ibm_watsonx_gov/evaluators/metrics_evaluator.py +187 -0
- ibm_watsonx_gov/evaluators/model_risk_evaluator.py +89 -0
- ibm_watsonx_gov/evaluators/traces_evaluator.py +93 -0
- ibm_watsonx_gov/metric_groups/answer_quality/answer_quality_decorator.py +66 -0
- ibm_watsonx_gov/metric_groups/content_safety/content_safety_decorator.py +76 -0
- ibm_watsonx_gov/metric_groups/readability/readability_decorator.py +59 -0
- ibm_watsonx_gov/metric_groups/retrieval_quality/retrieval_quality_decorator.py +63 -0
- ibm_watsonx_gov/metric_groups/usage/usage_decorator.py +58 -0
- ibm_watsonx_gov/metrics/__init__.py +74 -0
- ibm_watsonx_gov/metrics/answer_relevance/__init__.py +8 -0
- ibm_watsonx_gov/metrics/answer_relevance/answer_relevance_decorator.py +63 -0
- ibm_watsonx_gov/metrics/answer_relevance/answer_relevance_metric.py +260 -0
- ibm_watsonx_gov/metrics/answer_similarity/__init__.py +0 -0
- ibm_watsonx_gov/metrics/answer_similarity/answer_similarity_decorator.py +66 -0
- ibm_watsonx_gov/metrics/answer_similarity/answer_similarity_metric.py +219 -0
- ibm_watsonx_gov/metrics/average_precision/__init__.py +0 -0
- ibm_watsonx_gov/metrics/average_precision/average_precision_decorator.py +62 -0
- ibm_watsonx_gov/metrics/average_precision/average_precision_metric.py +174 -0
- ibm_watsonx_gov/metrics/base_metric_decorator.py +193 -0
- ibm_watsonx_gov/metrics/context_relevance/__init__.py +8 -0
- ibm_watsonx_gov/metrics/context_relevance/context_relevance_decorator.py +60 -0
- ibm_watsonx_gov/metrics/context_relevance/context_relevance_metric.py +414 -0
- ibm_watsonx_gov/metrics/cost/__init__.py +8 -0
- ibm_watsonx_gov/metrics/cost/cost_decorator.py +58 -0
- ibm_watsonx_gov/metrics/cost/cost_metric.py +155 -0
- ibm_watsonx_gov/metrics/duration/__init__.py +8 -0
- ibm_watsonx_gov/metrics/duration/duration_decorator.py +59 -0
- ibm_watsonx_gov/metrics/duration/duration_metric.py +111 -0
- ibm_watsonx_gov/metrics/evasiveness/__init__.py +8 -0
- ibm_watsonx_gov/metrics/evasiveness/evasiveness_decorator.py +61 -0
- ibm_watsonx_gov/metrics/evasiveness/evasiveness_metric.py +103 -0
- ibm_watsonx_gov/metrics/faithfulness/__init__.py +8 -0
- ibm_watsonx_gov/metrics/faithfulness/faithfulness_decorator.py +65 -0
- ibm_watsonx_gov/metrics/faithfulness/faithfulness_metric.py +254 -0
- ibm_watsonx_gov/metrics/hap/__init__.py +16 -0
- ibm_watsonx_gov/metrics/hap/hap_decorator.py +58 -0
- ibm_watsonx_gov/metrics/hap/hap_metric.py +98 -0
- ibm_watsonx_gov/metrics/hap/input_hap_metric.py +104 -0
- ibm_watsonx_gov/metrics/hap/output_hap_metric.py +110 -0
- ibm_watsonx_gov/metrics/harm/__init__.py +8 -0
- ibm_watsonx_gov/metrics/harm/harm_decorator.py +60 -0
- ibm_watsonx_gov/metrics/harm/harm_metric.py +103 -0
- ibm_watsonx_gov/metrics/harm_engagement/__init__.py +8 -0
- ibm_watsonx_gov/metrics/harm_engagement/harm_engagement_decorator.py +61 -0
- ibm_watsonx_gov/metrics/harm_engagement/harm_engagement_metric.py +103 -0
- ibm_watsonx_gov/metrics/hit_rate/__init__.py +0 -0
- ibm_watsonx_gov/metrics/hit_rate/hit_rate_decorator.py +59 -0
- ibm_watsonx_gov/metrics/hit_rate/hit_rate_metric.py +167 -0
- ibm_watsonx_gov/metrics/input_token_count/__init__.py +8 -0
- ibm_watsonx_gov/metrics/input_token_count/input_token_count_decorator.py +58 -0
- ibm_watsonx_gov/metrics/input_token_count/input_token_count_metric.py +112 -0
- ibm_watsonx_gov/metrics/jailbreak/__init__.py +8 -0
- ibm_watsonx_gov/metrics/jailbreak/jailbreak_decorator.py +60 -0
- ibm_watsonx_gov/metrics/jailbreak/jailbreak_metric.py +103 -0
- ibm_watsonx_gov/metrics/keyword_detection/keyword_detection_decorator.py +58 -0
- ibm_watsonx_gov/metrics/keyword_detection/keyword_detection_metric.py +111 -0
- ibm_watsonx_gov/metrics/llm_validation/__init__.py +8 -0
- ibm_watsonx_gov/metrics/llm_validation/evaluation_criteria.py +84 -0
- ibm_watsonx_gov/metrics/llm_validation/llm_validation_constants.py +24 -0
- ibm_watsonx_gov/metrics/llm_validation/llm_validation_decorator.py +54 -0
- ibm_watsonx_gov/metrics/llm_validation/llm_validation_impl.py +525 -0
- ibm_watsonx_gov/metrics/llm_validation/llm_validation_metric.py +258 -0
- ibm_watsonx_gov/metrics/llm_validation/llm_validation_prompts.py +106 -0
- ibm_watsonx_gov/metrics/llmaj/__init__.py +0 -0
- ibm_watsonx_gov/metrics/llmaj/llmaj_metric.py +298 -0
- ibm_watsonx_gov/metrics/ndcg/__init__.py +0 -0
- ibm_watsonx_gov/metrics/ndcg/ndcg_decorator.py +61 -0
- ibm_watsonx_gov/metrics/ndcg/ndcg_metric.py +166 -0
- ibm_watsonx_gov/metrics/output_token_count/__init__.py +8 -0
- ibm_watsonx_gov/metrics/output_token_count/output_token_count_decorator.py +58 -0
- ibm_watsonx_gov/metrics/output_token_count/output_token_count_metric.py +112 -0
- ibm_watsonx_gov/metrics/pii/__init__.py +16 -0
- ibm_watsonx_gov/metrics/pii/input_pii_metric.py +102 -0
- ibm_watsonx_gov/metrics/pii/output_pii_metric.py +107 -0
- ibm_watsonx_gov/metrics/pii/pii_decorator.py +59 -0
- ibm_watsonx_gov/metrics/pii/pii_metric.py +96 -0
- ibm_watsonx_gov/metrics/profanity/__init__.py +8 -0
- ibm_watsonx_gov/metrics/profanity/profanity_decorator.py +60 -0
- ibm_watsonx_gov/metrics/profanity/profanity_metric.py +103 -0
- ibm_watsonx_gov/metrics/prompt_safety_risk/__init__.py +8 -0
- ibm_watsonx_gov/metrics/prompt_safety_risk/prompt_safety_risk_decorator.py +57 -0
- ibm_watsonx_gov/metrics/prompt_safety_risk/prompt_safety_risk_metric.py +128 -0
- ibm_watsonx_gov/metrics/reciprocal_rank/__init__.py +0 -0
- ibm_watsonx_gov/metrics/reciprocal_rank/reciprocal_rank_decorator.py +62 -0
- ibm_watsonx_gov/metrics/reciprocal_rank/reciprocal_rank_metric.py +162 -0
- ibm_watsonx_gov/metrics/regex_detection/regex_detection_decorator.py +58 -0
- ibm_watsonx_gov/metrics/regex_detection/regex_detection_metric.py +106 -0
- ibm_watsonx_gov/metrics/retrieval_precision/__init__.py +0 -0
- ibm_watsonx_gov/metrics/retrieval_precision/retrieval_precision_decorator.py +62 -0
- ibm_watsonx_gov/metrics/retrieval_precision/retrieval_precision_metric.py +170 -0
- ibm_watsonx_gov/metrics/sexual_content/__init__.py +8 -0
- ibm_watsonx_gov/metrics/sexual_content/sexual_content_decorator.py +61 -0
- ibm_watsonx_gov/metrics/sexual_content/sexual_content_metric.py +103 -0
- ibm_watsonx_gov/metrics/social_bias/__init__.py +8 -0
- ibm_watsonx_gov/metrics/social_bias/social_bias_decorator.py +62 -0
- ibm_watsonx_gov/metrics/social_bias/social_bias_metric.py +103 -0
- ibm_watsonx_gov/metrics/status/__init__.py +0 -0
- ibm_watsonx_gov/metrics/status/status_metric.py +113 -0
- ibm_watsonx_gov/metrics/text_grade_level/__init__.py +8 -0
- ibm_watsonx_gov/metrics/text_grade_level/text_grade_level_decorator.py +59 -0
- ibm_watsonx_gov/metrics/text_grade_level/text_grade_level_metric.py +127 -0
- ibm_watsonx_gov/metrics/text_reading_ease/__init__.py +8 -0
- ibm_watsonx_gov/metrics/text_reading_ease/text_reading_ease_decorator.py +59 -0
- ibm_watsonx_gov/metrics/text_reading_ease/text_reading_ease_metric.py +123 -0
- ibm_watsonx_gov/metrics/tool_call_accuracy/__init__.py +0 -0
- ibm_watsonx_gov/metrics/tool_call_accuracy/tool_call_accuracy_decorator.py +67 -0
- ibm_watsonx_gov/metrics/tool_call_accuracy/tool_call_accuracy_metric.py +162 -0
- ibm_watsonx_gov/metrics/tool_call_parameter_accuracy/__init__.py +0 -0
- ibm_watsonx_gov/metrics/tool_call_parameter_accuracy/tool_call_parameter_accuracy_decorator.py +68 -0
- ibm_watsonx_gov/metrics/tool_call_parameter_accuracy/tool_call_parameter_accuracy_metric.py +151 -0
- ibm_watsonx_gov/metrics/tool_call_relevance/__init__.py +0 -0
- ibm_watsonx_gov/metrics/tool_call_relevance/tool_call_relevance_decorator.py +71 -0
- ibm_watsonx_gov/metrics/tool_call_relevance/tool_call_relevance_metric.py +166 -0
- ibm_watsonx_gov/metrics/tool_call_syntactic_accuracy/__init__.py +0 -0
- ibm_watsonx_gov/metrics/tool_call_syntactic_accuracy/tool_call_syntactic_accuracy_decorator.py +66 -0
- ibm_watsonx_gov/metrics/tool_call_syntactic_accuracy/tool_call_syntactic_accuracy_metric.py +121 -0
- ibm_watsonx_gov/metrics/topic_relevance/__init__.py +8 -0
- ibm_watsonx_gov/metrics/topic_relevance/topic_relevance_decorator.py +57 -0
- ibm_watsonx_gov/metrics/topic_relevance/topic_relevance_metric.py +106 -0
- ibm_watsonx_gov/metrics/unethical_behavior/__init__.py +8 -0
- ibm_watsonx_gov/metrics/unethical_behavior/unethical_behavior_decorator.py +61 -0
- ibm_watsonx_gov/metrics/unethical_behavior/unethical_behavior_metric.py +103 -0
- ibm_watsonx_gov/metrics/unsuccessful_requests/__init__.py +0 -0
- ibm_watsonx_gov/metrics/unsuccessful_requests/unsuccessful_requests_decorator.py +66 -0
- ibm_watsonx_gov/metrics/unsuccessful_requests/unsuccessful_requests_metric.py +128 -0
- ibm_watsonx_gov/metrics/user_id/__init__.py +0 -0
- ibm_watsonx_gov/metrics/user_id/user_id_metric.py +111 -0
- ibm_watsonx_gov/metrics/utils.py +440 -0
- ibm_watsonx_gov/metrics/violence/__init__.py +8 -0
- ibm_watsonx_gov/metrics/violence/violence_decorator.py +60 -0
- ibm_watsonx_gov/metrics/violence/violence_metric.py +103 -0
- ibm_watsonx_gov/prompt_evaluator/__init__.py +9 -0
- ibm_watsonx_gov/prompt_evaluator/impl/__init__.py +8 -0
- ibm_watsonx_gov/prompt_evaluator/impl/prompt_evaluator_impl.py +554 -0
- ibm_watsonx_gov/prompt_evaluator/impl/pta_lifecycle_evaluator.py +2332 -0
- ibm_watsonx_gov/prompt_evaluator/prompt_evaluator.py +262 -0
- ibm_watsonx_gov/providers/__init__.py +8 -0
- ibm_watsonx_gov/providers/detectors_provider.cp313-win_amd64.pyd +0 -0
- ibm_watsonx_gov/providers/detectors_provider.py +415 -0
- ibm_watsonx_gov/providers/eval_assist_provider.cp313-win_amd64.pyd +0 -0
- ibm_watsonx_gov/providers/eval_assist_provider.py +266 -0
- ibm_watsonx_gov/providers/inference_engines/__init__.py +0 -0
- ibm_watsonx_gov/providers/inference_engines/custom_inference_engine.py +165 -0
- ibm_watsonx_gov/providers/inference_engines/portkey_inference_engine.py +57 -0
- ibm_watsonx_gov/providers/llmevalkit/__init__.py +0 -0
- ibm_watsonx_gov/providers/llmevalkit/ciso_agent/main.py +516 -0
- ibm_watsonx_gov/providers/llmevalkit/ciso_agent/preprocess_log.py +111 -0
- ibm_watsonx_gov/providers/llmevalkit/ciso_agent/utils.py +186 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/README.md +411 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/__init__.py +27 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/comparison/README.md +306 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/comparison/__init__.py +89 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/comparison/comparators/__init__.py +30 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/comparison/comparators/base.py +411 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/comparison/comparators/code_agent.py +1254 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/comparison/comparators/exact_match.py +134 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/comparison/comparators/fuzzy_string.py +104 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/comparison/comparators/hybrid.py +516 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/comparison/comparators/llm_judge.py +1882 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/comparison/pipeline.py +387 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/comparison/types.py +178 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/comparison/utils.py +298 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/consts.py +33 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/__init__.py +31 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/base.py +26 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/function_call/__init__.py +4 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/function_call/general.py +46 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/function_call/general_metrics.json +783 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/function_call/general_metrics_runtime.json +580 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/function_selection/__init__.py +6 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/function_selection/function_selection.py +28 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/function_selection/function_selection_metrics.json +599 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/function_selection/function_selection_metrics_runtime.json +477 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/loader.py +259 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/parameter/__init__.py +7 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/parameter/parameter.py +52 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/parameter/parameter_metrics.json +613 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/parameter/parameter_metrics_runtime.json +489 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/trajectory/__init__.py +7 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/trajectory/trajectory.py +43 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/trajectory/trajectory_metrics.json +161 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/pipeline/__init__.py +0 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/pipeline/adapters.py +102 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/pipeline/pipeline.py +355 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/pipeline/semantic_checker.py +816 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/pipeline/static_checker.py +297 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/pipeline/transformation_prompts.py +509 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/pipeline/types.py +596 -0
- ibm_watsonx_gov/providers/llmevalkit/llm/README.md +375 -0
- ibm_watsonx_gov/providers/llmevalkit/llm/__init__.py +137 -0
- ibm_watsonx_gov/providers/llmevalkit/llm/base.py +426 -0
- ibm_watsonx_gov/providers/llmevalkit/llm/output_parser.py +364 -0
- ibm_watsonx_gov/providers/llmevalkit/llm/providers/__init__.py +0 -0
- ibm_watsonx_gov/providers/llmevalkit/llm/providers/consts.py +7 -0
- ibm_watsonx_gov/providers/llmevalkit/llm/providers/ibm_watsonx_ai/__init__.py +0 -0
- ibm_watsonx_gov/providers/llmevalkit/llm/providers/ibm_watsonx_ai/ibm_watsonx_ai.py +656 -0
- ibm_watsonx_gov/providers/llmevalkit/llm/providers/litellm/__init__.py +0 -0
- ibm_watsonx_gov/providers/llmevalkit/llm/providers/litellm/litellm.py +509 -0
- ibm_watsonx_gov/providers/llmevalkit/llm/providers/litellm/rits.py +224 -0
- ibm_watsonx_gov/providers/llmevalkit/llm/providers/litellm/watsonx.py +60 -0
- ibm_watsonx_gov/providers/llmevalkit/llm/providers/mock_llm_client.py +75 -0
- ibm_watsonx_gov/providers/llmevalkit/llm/providers/openai/__init__.py +0 -0
- ibm_watsonx_gov/providers/llmevalkit/llm/providers/openai/openai.py +639 -0
- ibm_watsonx_gov/providers/llmevalkit/llm/providers/wxo_ai_gateway/__init__.py +0 -0
- ibm_watsonx_gov/providers/llmevalkit/llm/providers/wxo_ai_gateway/wxo_ai_gateway.py +134 -0
- ibm_watsonx_gov/providers/llmevalkit/llm/providers/wxo_ai_gateway/wxo_ai_gateway_inference.py +214 -0
- ibm_watsonx_gov/providers/llmevalkit/llm/types.py +136 -0
- ibm_watsonx_gov/providers/llmevalkit/metrics/__init__.py +4 -0
- ibm_watsonx_gov/providers/llmevalkit/metrics/field.py +255 -0
- ibm_watsonx_gov/providers/llmevalkit/metrics/metric.py +332 -0
- ibm_watsonx_gov/providers/llmevalkit/metrics/metrics_runner.py +188 -0
- ibm_watsonx_gov/providers/llmevalkit/metrics/prompt.py +403 -0
- ibm_watsonx_gov/providers/llmevalkit/metrics/utils.py +46 -0
- ibm_watsonx_gov/providers/llmevalkit/prompt/__init__.py +0 -0
- ibm_watsonx_gov/providers/llmevalkit/prompt/runner.py +144 -0
- ibm_watsonx_gov/providers/tool_call_metric_provider.py +455 -0
- ibm_watsonx_gov/providers/unitxt_provider.cp313-win_amd64.pyd +0 -0
- ibm_watsonx_gov/tools/__init__.py +10 -0
- ibm_watsonx_gov/tools/clients/__init__.py +11 -0
- ibm_watsonx_gov/tools/clients/ai_tool_client.py +405 -0
- ibm_watsonx_gov/tools/clients/detector_client.py +82 -0
- ibm_watsonx_gov/tools/core/__init__.py +8 -0
- ibm_watsonx_gov/tools/core/tool_loader.py +237 -0
- ibm_watsonx_gov/tools/entities/__init__.py +8 -0
- ibm_watsonx_gov/tools/entities/ai_tools.py +435 -0
- ibm_watsonx_gov/tools/onboarding/create/answer_relevance_detector.json +57 -0
- ibm_watsonx_gov/tools/onboarding/create/chromadb_retrieval_tool.json +63 -0
- ibm_watsonx_gov/tools/onboarding/create/context_relevance_detector.json +57 -0
- ibm_watsonx_gov/tools/onboarding/create/duduckgo_search_tool.json +53 -0
- ibm_watsonx_gov/tools/onboarding/create/google_search_tool.json +62 -0
- ibm_watsonx_gov/tools/onboarding/create/hap_detector.json +70 -0
- ibm_watsonx_gov/tools/onboarding/create/jailbreak_detector.json +70 -0
- ibm_watsonx_gov/tools/onboarding/create/pii_detector.json +36 -0
- ibm_watsonx_gov/tools/onboarding/create/prompt_safety_risk_detector.json +69 -0
- ibm_watsonx_gov/tools/onboarding/create/topic_relevance_detector.json +57 -0
- ibm_watsonx_gov/tools/onboarding/create/weather_tool.json +39 -0
- ibm_watsonx_gov/tools/onboarding/create/webcrawler_tool.json +34 -0
- ibm_watsonx_gov/tools/onboarding/create/wikipedia_search_tool.json +53 -0
- ibm_watsonx_gov/tools/onboarding/delete/delete_tools.json +4 -0
- ibm_watsonx_gov/tools/onboarding/update/google_search_tool.json +38 -0
- ibm_watsonx_gov/tools/ootb/__init__.py +8 -0
- ibm_watsonx_gov/tools/ootb/detectors/__init__.py +8 -0
- ibm_watsonx_gov/tools/ootb/detectors/hap_detector_tool.py +109 -0
- ibm_watsonx_gov/tools/ootb/detectors/jailbreak_detector_tool.py +104 -0
- ibm_watsonx_gov/tools/ootb/detectors/pii_detector_tool.py +83 -0
- ibm_watsonx_gov/tools/ootb/detectors/prompt_safety_risk_detector_tool.py +111 -0
- ibm_watsonx_gov/tools/ootb/detectors/topic_relevance_detector_tool.py +101 -0
- ibm_watsonx_gov/tools/ootb/rag/__init__.py +8 -0
- ibm_watsonx_gov/tools/ootb/rag/answer_relevance_detector_tool.py +119 -0
- ibm_watsonx_gov/tools/ootb/rag/context_relevance_detector_tool.py +118 -0
- ibm_watsonx_gov/tools/ootb/search/__init__.py +8 -0
- ibm_watsonx_gov/tools/ootb/search/duckduckgo_search_tool.py +62 -0
- ibm_watsonx_gov/tools/ootb/search/google_search_tool.py +105 -0
- ibm_watsonx_gov/tools/ootb/search/weather_tool.py +95 -0
- ibm_watsonx_gov/tools/ootb/search/web_crawler_tool.py +69 -0
- ibm_watsonx_gov/tools/ootb/search/wikipedia_search_tool.py +63 -0
- ibm_watsonx_gov/tools/ootb/vectordb/__init__.py +8 -0
- ibm_watsonx_gov/tools/ootb/vectordb/chromadb_retriever_tool.py +111 -0
- ibm_watsonx_gov/tools/rest_api/__init__.py +10 -0
- ibm_watsonx_gov/tools/rest_api/restapi_tool.py +72 -0
- ibm_watsonx_gov/tools/schemas/__init__.py +10 -0
- ibm_watsonx_gov/tools/schemas/search_tool_schema.py +46 -0
- ibm_watsonx_gov/tools/schemas/vectordb_retrieval_schema.py +55 -0
- ibm_watsonx_gov/tools/utils/__init__.py +14 -0
- ibm_watsonx_gov/tools/utils/constants.py +69 -0
- ibm_watsonx_gov/tools/utils/display_utils.py +38 -0
- ibm_watsonx_gov/tools/utils/environment.py +108 -0
- ibm_watsonx_gov/tools/utils/package_utils.py +40 -0
- ibm_watsonx_gov/tools/utils/platform_url_mapping.cp313-win_amd64.pyd +0 -0
- ibm_watsonx_gov/tools/utils/python_utils.py +68 -0
- ibm_watsonx_gov/tools/utils/tool_utils.py +206 -0
- ibm_watsonx_gov/traces/__init__.py +8 -0
- ibm_watsonx_gov/traces/span_exporter.py +195 -0
- ibm_watsonx_gov/traces/span_node.py +251 -0
- ibm_watsonx_gov/traces/span_util.py +153 -0
- ibm_watsonx_gov/traces/trace_utils.py +1074 -0
- ibm_watsonx_gov/utils/__init__.py +8 -0
- ibm_watsonx_gov/utils/aggregation_util.py +346 -0
- ibm_watsonx_gov/utils/async_util.py +62 -0
- ibm_watsonx_gov/utils/authenticator.py +144 -0
- ibm_watsonx_gov/utils/constants.py +15 -0
- ibm_watsonx_gov/utils/errors.py +40 -0
- ibm_watsonx_gov/utils/gov_sdk_logger.py +39 -0
- ibm_watsonx_gov/utils/insights_generator.py +1285 -0
- ibm_watsonx_gov/utils/python_utils.py +425 -0
- ibm_watsonx_gov/utils/rest_util.py +73 -0
- ibm_watsonx_gov/utils/segment_batch_manager.py +162 -0
- ibm_watsonx_gov/utils/singleton_meta.py +25 -0
- ibm_watsonx_gov/utils/url_mapping.cp313-win_amd64.pyd +0 -0
- ibm_watsonx_gov/utils/validation_util.py +126 -0
- ibm_watsonx_gov/visualizations/__init__.py +13 -0
- ibm_watsonx_gov/visualizations/metric_descriptions.py +57 -0
- ibm_watsonx_gov/visualizations/model_insights.py +1304 -0
- ibm_watsonx_gov/visualizations/visualization_utils.py +75 -0
- ibm_watsonx_gov-1.3.3.dist-info/METADATA +93 -0
- ibm_watsonx_gov-1.3.3.dist-info/RECORD +353 -0
- ibm_watsonx_gov-1.3.3.dist-info/WHEEL +4 -0
|
@@ -0,0 +1,980 @@
|
|
|
1
|
+
# ----------------------------------------------------------------------------------------------------
|
|
2
|
+
# IBM Confidential
|
|
3
|
+
# Licensed Materials - Property of IBM
|
|
4
|
+
# 5737-H76, 5900-A3Q
|
|
5
|
+
# © Copyright IBM Corp. 2025 All Rights Reserved.
|
|
6
|
+
# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
|
|
7
|
+
# GSA ADPSchedule Contract with IBM Corp.
|
|
8
|
+
# ----------------------------------------------------------------------------------------------------
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
import os
|
|
12
|
+
import time
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Any, Dict, List, Optional
|
|
15
|
+
|
|
16
|
+
import nbformat
|
|
17
|
+
import requests
|
|
18
|
+
|
|
19
|
+
from ibm_watsonx_gov.agent_catalog.utils.notebook_utils import \
|
|
20
|
+
get_all_code_from_notebook
|
|
21
|
+
from ibm_watsonx_gov.ai_experiments.utils.ai_experiment_utils import \
|
|
22
|
+
AIExperimentUtils
|
|
23
|
+
from ibm_watsonx_gov.clients.api_client import APIClient
|
|
24
|
+
from ibm_watsonx_gov.entities.ai_evaluation import (AIEvaluationAsset,
|
|
25
|
+
EvaluationConfig)
|
|
26
|
+
from ibm_watsonx_gov.entities.ai_experiment import (AIExperiment,
|
|
27
|
+
AIExperimentRun)
|
|
28
|
+
from ibm_watsonx_gov.utils.gov_sdk_logger import GovSDKLogger
|
|
29
|
+
from ibm_watsonx_gov.utils.python_utils import get, get_authenticator_token
|
|
30
|
+
from ibm_watsonx_gov.utils.rest_util import RestUtil
|
|
31
|
+
from ibm_watsonx_gov.utils.url_mapping import WOS_URL_MAPPING
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class AIExperimentsClient:
|
|
35
|
+
|
|
36
|
+
def __init__(
|
|
37
|
+
self, api_client: APIClient, project_id: str = None, space_id: str = None
|
|
38
|
+
) -> None:
|
|
39
|
+
"""
|
|
40
|
+
Initialize the AIExperimentsClient class.
|
|
41
|
+
|
|
42
|
+
Args:
|
|
43
|
+
- api_client: The watsonx.governance client, used for authentication.
|
|
44
|
+
- project_id: The project id.
|
|
45
|
+
- space_id: The space id.
|
|
46
|
+
|
|
47
|
+
Example:
|
|
48
|
+
-------
|
|
49
|
+
Initialize AIExperimentsClient:
|
|
50
|
+
.. code-block:: python
|
|
51
|
+
|
|
52
|
+
# Initialize the API client
|
|
53
|
+
api_client = APIClient(credentials=Credentials(api_key="", url="wos_url"))
|
|
54
|
+
|
|
55
|
+
# Create the AI Experiment client
|
|
56
|
+
ai_experiment_client = AIExperimentClient(api_client=api_client, project_id="your_project_id")
|
|
57
|
+
"""
|
|
58
|
+
self.logger = GovSDKLogger.get_logger(__name__)
|
|
59
|
+
if not api_client:
|
|
60
|
+
api_client = APIClient()
|
|
61
|
+
self.api_client = api_client
|
|
62
|
+
self.dataplatform_url = api_client.credentials.url
|
|
63
|
+
# Fetching URL map to get dataplatform url for cloud environments
|
|
64
|
+
if not self.api_client.is_cpd:
|
|
65
|
+
url_map = WOS_URL_MAPPING.get(api_client.credentials.url)
|
|
66
|
+
self.dataplatform_url = url_map.dai_url
|
|
67
|
+
|
|
68
|
+
self.ai_experiment_url = (
|
|
69
|
+
f"{self.dataplatform_url}/v1/aigov/factsheet/ai_experiments"
|
|
70
|
+
)
|
|
71
|
+
self.project_id = project_id
|
|
72
|
+
self.verify_ssl = not self.api_client.credentials.disable_ssl
|
|
73
|
+
# container checks
|
|
74
|
+
self.__container_checks(project_id, space_id)
|
|
75
|
+
|
|
76
|
+
def create(self, experiment_details: AIExperiment) -> AIExperiment:
|
|
77
|
+
"""
|
|
78
|
+
Creates AI experiment asset with specified details
|
|
79
|
+
|
|
80
|
+
Args:
|
|
81
|
+
- experiment_details: The instance of AIExperiment having details of the experiment to be created.
|
|
82
|
+
|
|
83
|
+
Returns: An instance of AIExperiment.
|
|
84
|
+
|
|
85
|
+
Examples:
|
|
86
|
+
---------
|
|
87
|
+
Create an AI Experiment:
|
|
88
|
+
.. code-block:: python
|
|
89
|
+
|
|
90
|
+
# Initialize the API client with credentials
|
|
91
|
+
api_client = APIClient(credentials=Credentials(api_key="", url=""))
|
|
92
|
+
|
|
93
|
+
# Create the AI Experiment client with your project ID
|
|
94
|
+
ai_experiment_client = AIExperimentClient(api_client=api_client, project_id="your_project_id")
|
|
95
|
+
|
|
96
|
+
# Create the AIExperiment instance
|
|
97
|
+
ai_experiment = AIExperiment(name="", description="", component_type="agent", component_name="")
|
|
98
|
+
|
|
99
|
+
ai_experiment_asset = ai_experiment_client.create(ai_experiment)
|
|
100
|
+
"""
|
|
101
|
+
|
|
102
|
+
ai_experiment_payload = experiment_details.to_json()
|
|
103
|
+
|
|
104
|
+
response = RestUtil.request_with_retry().post(
|
|
105
|
+
self.ai_experiment_url,
|
|
106
|
+
json=ai_experiment_payload,
|
|
107
|
+
headers=self.__get_headers(),
|
|
108
|
+
params=self.container_params,
|
|
109
|
+
verify=self.verify_ssl
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
if not response.ok:
|
|
113
|
+
message = f"Error occurred while creating AI experiment asset. Status code: {response.status_code}, Error: {response.text}"
|
|
114
|
+
self.logger.error(message)
|
|
115
|
+
raise Exception(message)
|
|
116
|
+
|
|
117
|
+
response_json = response.json()
|
|
118
|
+
|
|
119
|
+
ai_experiment = self.__map_ai_experiment(response_json)
|
|
120
|
+
|
|
121
|
+
ai_experiment_id = response_json.get("asset_id")
|
|
122
|
+
print(f"Created AI experiment asset with id {ai_experiment_id}.\n")
|
|
123
|
+
return ai_experiment
|
|
124
|
+
|
|
125
|
+
def get(self, ai_experiment_id: str) -> AIExperiment:
|
|
126
|
+
"""
|
|
127
|
+
Retrieves AI experiment asset details
|
|
128
|
+
Args:
|
|
129
|
+
- ai_experiment_id: The ID of AI experiment asset.
|
|
130
|
+
Returns: An instance of AIExperiment.
|
|
131
|
+
"""
|
|
132
|
+
ai_experiment_url = f"{self.ai_experiment_url}/{ai_experiment_id}"
|
|
133
|
+
response = RestUtil.request_with_retry().get(
|
|
134
|
+
ai_experiment_url,
|
|
135
|
+
headers=self.__get_headers(),
|
|
136
|
+
params=self.container_params,
|
|
137
|
+
verify=self.verify_ssl
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
if not response.ok:
|
|
141
|
+
message = f"Error occurred while retrieving AI experiment asset. Status code: {response.status_code}, Error: {response.text}"
|
|
142
|
+
self.logger.error(message)
|
|
143
|
+
raise Exception(message)
|
|
144
|
+
|
|
145
|
+
response_json = response.json()
|
|
146
|
+
|
|
147
|
+
asset_details = response_json.pop("asset_details", {})
|
|
148
|
+
ai_experiment_runs = response_json.pop("ai_experiment_runs", [])
|
|
149
|
+
|
|
150
|
+
ai_experiment_response = {
|
|
151
|
+
**response_json,
|
|
152
|
+
**asset_details,
|
|
153
|
+
"runs": ai_experiment_runs,
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
ai_experiment = AIExperiment(**ai_experiment_response)
|
|
157
|
+
|
|
158
|
+
print(f"Retrieved AI experiment asset {ai_experiment_id}.\n")
|
|
159
|
+
return ai_experiment
|
|
160
|
+
|
|
161
|
+
def search(self, ai_experiment_name: str) -> AIExperiment:
|
|
162
|
+
"""
|
|
163
|
+
Searches AI experiment with specified name
|
|
164
|
+
Args:
|
|
165
|
+
- ai_experiment_name: The name of AI experiment to be searched.
|
|
166
|
+
Returns: An instance of AIExperiment.
|
|
167
|
+
"""
|
|
168
|
+
ai_experiment = None
|
|
169
|
+
# Search using asset search API
|
|
170
|
+
search_url = f"{self.dataplatform_url}/v2/asset_types/ai_experiment/search"
|
|
171
|
+
|
|
172
|
+
search_payload = {
|
|
173
|
+
"query": f"asset.name:\"{ai_experiment_name}\"",
|
|
174
|
+
"sort": "asset.name<string>"
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
response = RestUtil.request_with_retry().post(
|
|
178
|
+
search_url,
|
|
179
|
+
json=search_payload,
|
|
180
|
+
headers=self.__get_headers(),
|
|
181
|
+
params=self.container_params,
|
|
182
|
+
verify=self.verify_ssl
|
|
183
|
+
)
|
|
184
|
+
if not response.ok:
|
|
185
|
+
message = f"Error occurred while searching AI experiment with name {ai_experiment_name}. Status code: {response.status_code}, Error: {response.text}"
|
|
186
|
+
self.logger.error(message)
|
|
187
|
+
raise Exception(message)
|
|
188
|
+
|
|
189
|
+
response_json = response.json()
|
|
190
|
+
if response_json.get("results"):
|
|
191
|
+
ai_experiment_details = response_json.get("results")[0]
|
|
192
|
+
ai_experiment = AIExperiment(
|
|
193
|
+
**ai_experiment_details.get("metadata"))
|
|
194
|
+
print(
|
|
195
|
+
f"Using existing AI experiment with id {ai_experiment.asset_id}.\n")
|
|
196
|
+
|
|
197
|
+
return ai_experiment
|
|
198
|
+
|
|
199
|
+
def update(
|
|
200
|
+
self,
|
|
201
|
+
ai_experiment_id: str,
|
|
202
|
+
experiment_run_details: AIExperimentRun,
|
|
203
|
+
evaluation_results=None,
|
|
204
|
+
track_notebook=False,
|
|
205
|
+
) -> AIExperiment:
|
|
206
|
+
"""
|
|
207
|
+
Updates AI experiment asset details, with the given experiment run details.
|
|
208
|
+
Args:
|
|
209
|
+
- ai_experiment_id: The ID of AI experiment asset to be updated
|
|
210
|
+
- experiment_run_details : An instance of AIExperimentRun, payload to create attachment
|
|
211
|
+
- evaluation_result:(DataFrame|ToolMetricResult) The content of attachment to be uploaded as file (Optional)
|
|
212
|
+
- track_notebook:(bool) If set to True the notebook will be stored as attachment
|
|
213
|
+
Returns: The updated AI experiment asset details
|
|
214
|
+
|
|
215
|
+
Examples:
|
|
216
|
+
-----------
|
|
217
|
+
Updating a AI Experiment with the evaluation results:
|
|
218
|
+
.. code-block:: python
|
|
219
|
+
|
|
220
|
+
# Initialize the API client with credentials
|
|
221
|
+
api_client = APIClient(credentials=Credentials(api_key="", url="wos_url"))
|
|
222
|
+
|
|
223
|
+
# Create the AI Experiment client with your project ID
|
|
224
|
+
ai_experiment_client = AIExperimentClient(api_client=api_client, project_id="your_project_id")
|
|
225
|
+
|
|
226
|
+
# Define ai_experiment_runs
|
|
227
|
+
experiment_run_details = AIExperimentRun(run_id=str(uuid.uuid4()), run_name="", test_data={}, node=[])
|
|
228
|
+
|
|
229
|
+
# evaluation_result will be an instance of ToolMetricResult or DataFrame
|
|
230
|
+
|
|
231
|
+
# Update the AI experiment asset with run results
|
|
232
|
+
updated_ai_experiment_details = ai_experiment_client.update(
|
|
233
|
+
ai_experiment_asset_id="",
|
|
234
|
+
experiment_run_details=experiment_run_details,
|
|
235
|
+
evaluation_result=run_result
|
|
236
|
+
)
|
|
237
|
+
|
|
238
|
+
"""
|
|
239
|
+
ai_experiment_url = f"{self.ai_experiment_url}/{ai_experiment_id}"
|
|
240
|
+
|
|
241
|
+
attachment_id = None
|
|
242
|
+
notebook_attachment_id = None
|
|
243
|
+
notebook_file_path = None
|
|
244
|
+
code_attachment_id = None
|
|
245
|
+
# If evaluation_result is not None, this method will upload the result and create an attachment for it.
|
|
246
|
+
if evaluation_results:
|
|
247
|
+
# Convert the evaluator result into attachment format.
|
|
248
|
+
result_attachment, total_records = AIExperimentUtils.construct_result_attachment_payload(
|
|
249
|
+
evaluation_results, experiment_run_details.nodes)
|
|
250
|
+
attachment_id = self.__store_experiment_run_result(
|
|
251
|
+
ai_experiment_id, experiment_run_details, result_attachment
|
|
252
|
+
)
|
|
253
|
+
|
|
254
|
+
# Determine notebook path with fallback logic
|
|
255
|
+
if track_notebook:
|
|
256
|
+
if experiment_run_details.source_url:
|
|
257
|
+
notebook_file_path = Path(experiment_run_details.source_url)
|
|
258
|
+
elif experiment_run_details.source_name:
|
|
259
|
+
notebook_file_path = Path(
|
|
260
|
+
experiment_run_details.source_name).expanduser().resolve()
|
|
261
|
+
else:
|
|
262
|
+
track_notebook = False
|
|
263
|
+
|
|
264
|
+
if track_notebook:
|
|
265
|
+
try:
|
|
266
|
+
notebook = AIExperimentsClient.__process_notebook(
|
|
267
|
+
notebook_file_path)
|
|
268
|
+
notebook_payload = {
|
|
269
|
+
"asset_type": "ai_experiment",
|
|
270
|
+
"name": f"AI experiment notebook for {experiment_run_details.run_id}",
|
|
271
|
+
"description": f"AI experiment notebook for {experiment_run_details.run_name}, notebook name: {experiment_run_details.source_name}",
|
|
272
|
+
"mime": "json/txt",
|
|
273
|
+
"user_data": {},
|
|
274
|
+
}
|
|
275
|
+
notebook_attachment_id = self.__create_asset_attachment(
|
|
276
|
+
ai_experiment_id, notebook_payload, notebook
|
|
277
|
+
).get("attachment_id")
|
|
278
|
+
except Exception as e:
|
|
279
|
+
message = f"Failed to track the notebook for experiment run {experiment_run_details.run_id}. Error: {str(e)}"
|
|
280
|
+
self.logger.error(message)
|
|
281
|
+
raise Exception(message)
|
|
282
|
+
|
|
283
|
+
if experiment_run_details.source_name and experiment_run_details.agent_method_name:
|
|
284
|
+
# extract code and store with experiment run
|
|
285
|
+
code = get_all_code_from_notebook(
|
|
286
|
+
experiment_run_details.source_name, experiment_run_details.agent_method_name)
|
|
287
|
+
try:
|
|
288
|
+
code_payload = {
|
|
289
|
+
"asset_type": "ai_experiment",
|
|
290
|
+
"name": f"Agent code for {experiment_run_details.run_id}",
|
|
291
|
+
"description": f"Agent code for {experiment_run_details.run_name}",
|
|
292
|
+
"mime": "json/txt",
|
|
293
|
+
"user_data": {},
|
|
294
|
+
}
|
|
295
|
+
code_attachment_id = self.__create_asset_attachment(
|
|
296
|
+
ai_experiment_id, code_payload, code
|
|
297
|
+
).get("attachment_id")
|
|
298
|
+
except Exception as e:
|
|
299
|
+
message = f"Failed to track the code for experiment run {experiment_run_details.run_id}. Error: {str(e)}"
|
|
300
|
+
self.logger.error(message)
|
|
301
|
+
raise Exception(message)
|
|
302
|
+
|
|
303
|
+
test_data = experiment_run_details.test_data or {}
|
|
304
|
+
test_data["total_rows"] = total_records
|
|
305
|
+
# Updating run details in the AI experiment asset
|
|
306
|
+
update_payload = [{
|
|
307
|
+
"run_id": experiment_run_details.run_id,
|
|
308
|
+
"run_name": experiment_run_details.run_name,
|
|
309
|
+
"test_data": test_data,
|
|
310
|
+
"nodes": [node.to_json() for node in experiment_run_details.nodes] or [],
|
|
311
|
+
"attachment_id": attachment_id,
|
|
312
|
+
"source_url": experiment_run_details.source_url,
|
|
313
|
+
"source_name": experiment_run_details.source_name,
|
|
314
|
+
"duration": experiment_run_details.duration,
|
|
315
|
+
"custom_tags": experiment_run_details.custom_tags,
|
|
316
|
+
"properties": {
|
|
317
|
+
"notebook_attachment_id": notebook_attachment_id,
|
|
318
|
+
"code_attachment_id": code_attachment_id,
|
|
319
|
+
}
|
|
320
|
+
}]
|
|
321
|
+
|
|
322
|
+
response = RestUtil.request_with_retry().put(
|
|
323
|
+
ai_experiment_url,
|
|
324
|
+
json=update_payload,
|
|
325
|
+
headers=self.__get_headers(),
|
|
326
|
+
params=self.container_params,
|
|
327
|
+
verify=self.verify_ssl
|
|
328
|
+
)
|
|
329
|
+
|
|
330
|
+
if not response.ok:
|
|
331
|
+
message = f"Error occurred while updating AI experiment asset. Status code: {response.status_code}, Error: {response.text}"
|
|
332
|
+
self.logger.error(message)
|
|
333
|
+
raise Exception(message)
|
|
334
|
+
|
|
335
|
+
print(
|
|
336
|
+
f"Updated experiment run details for run {experiment_run_details.run_name} of AI experiment {ai_experiment_id}.\n"
|
|
337
|
+
)
|
|
338
|
+
|
|
339
|
+
response_json = response.json()
|
|
340
|
+
|
|
341
|
+
ai_experiment = AIExperiment(**response_json)
|
|
342
|
+
|
|
343
|
+
print(f"Updated AI experiment asset {ai_experiment_id}.\n")
|
|
344
|
+
return ai_experiment
|
|
345
|
+
|
|
346
|
+
def create_ai_evaluation_asset(
|
|
347
|
+
self,
|
|
348
|
+
ai_experiment_ids: List[str] = None,
|
|
349
|
+
ai_experiment_runs: Dict[str, List[AIExperimentRun]] = None,
|
|
350
|
+
ai_evaluation_details: AIEvaluationAsset = None,
|
|
351
|
+
) -> AIEvaluationAsset:
|
|
352
|
+
"""
|
|
353
|
+
Creates an AI Evaluation asset from either experiment IDs or experiment run mappings.
|
|
354
|
+
|
|
355
|
+
Args:
|
|
356
|
+
ai_experiment_ids (List[str], optional):
|
|
357
|
+
A list of AI experiment IDs for which the evaluation asset should be created.
|
|
358
|
+
ai_experiment_runs (Dict[str, List[AIExperimentRun]], optional):
|
|
359
|
+
A list of dictionaries where each dictionary maps an experiment ID (str)
|
|
360
|
+
to an AIExperimentRun object.
|
|
361
|
+
ai_evaluation_details (AIEvaluationAsset, optional):
|
|
362
|
+
An instance of AIEvaluationAsset having details (name, description and metrics configuration)
|
|
363
|
+
Returns:
|
|
364
|
+
An instance of AIEvaluationAsset.
|
|
365
|
+
|
|
366
|
+
Note:
|
|
367
|
+
Only one of `ai_experiment_ids` or `ai_experiment_runs` should be provided.
|
|
368
|
+
|
|
369
|
+
Examples:
|
|
370
|
+
-----------
|
|
371
|
+
Comparing a list of AI experiments:
|
|
372
|
+
.. code-block:: python
|
|
373
|
+
|
|
374
|
+
# Initialize the API client with credentials
|
|
375
|
+
api_client = APIClient(credentials=Credentials(api_key="", url="wos_url"))
|
|
376
|
+
|
|
377
|
+
# Create the AI Experiment client with your project ID
|
|
378
|
+
ai_experiment_client = AIExperimentClient(api_client=api_client, project_id="your_project_id")
|
|
379
|
+
|
|
380
|
+
# Create AI Experiments
|
|
381
|
+
ai_experiment = ai_experiment_client.create(name="",description="",component_type="",component_name="")
|
|
382
|
+
|
|
383
|
+
# Define evaluation configuration
|
|
384
|
+
evaluation_config = EvaluationConfig(
|
|
385
|
+
monitors={
|
|
386
|
+
"agentic_ai_quality": {
|
|
387
|
+
"parameters": {
|
|
388
|
+
"metrics_configuration": {}
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
)
|
|
393
|
+
|
|
394
|
+
# Create the evaluation asset
|
|
395
|
+
ai_evaluation_asset = AIEvaluationAsset(
|
|
396
|
+
name="AI Evaluation for agent",
|
|
397
|
+
evaluation_configuration=evaluation_config
|
|
398
|
+
)
|
|
399
|
+
|
|
400
|
+
# Compare two or more AI experiments using the evaluation asset
|
|
401
|
+
response = ai_experiment_client.compare_ai_experiments(
|
|
402
|
+
ai_experiment_ids=["experiment_id_1", "experiment_id_2"],
|
|
403
|
+
ai_evaluation_asset=ai_evaluation_asset
|
|
404
|
+
)
|
|
405
|
+
# Link for AIEvaluationAsset
|
|
406
|
+
response.href
|
|
407
|
+
"""
|
|
408
|
+
|
|
409
|
+
if ai_experiment_ids and ai_experiment_runs:
|
|
410
|
+
message = f"Both 'ai_experiment_ids' and 'ai_experiment_runs' cannot be passed together."
|
|
411
|
+
self.logger.error(message)
|
|
412
|
+
raise Exception(message)
|
|
413
|
+
|
|
414
|
+
start_time = time.time()
|
|
415
|
+
time_str = time.strftime(
|
|
416
|
+
'%Y-%m-%d %H:%M:%S', time.localtime(start_time))
|
|
417
|
+
|
|
418
|
+
evaluation_assets = []
|
|
419
|
+
total_runs = 0
|
|
420
|
+
|
|
421
|
+
if ai_experiment_ids:
|
|
422
|
+
for experiment_id in ai_experiment_ids:
|
|
423
|
+
ai_experiment = self.get(ai_experiment_id=experiment_id)
|
|
424
|
+
experiment_name = ai_experiment.name
|
|
425
|
+
runs_data = ai_experiment.runs
|
|
426
|
+
for run in runs_data:
|
|
427
|
+
evaluation_assets.append(
|
|
428
|
+
self.__build_evaluation_asset(
|
|
429
|
+
experiment_id, experiment_name, run.to_json()
|
|
430
|
+
)
|
|
431
|
+
)
|
|
432
|
+
|
|
433
|
+
elif ai_experiment_runs:
|
|
434
|
+
for experiment_id, runs in ai_experiment_runs.items():
|
|
435
|
+
ai_experiment = self.get(ai_experiment_id=experiment_id)
|
|
436
|
+
experiment_name = ai_experiment.name
|
|
437
|
+
if not runs:
|
|
438
|
+
runs = ai_experiment.runs
|
|
439
|
+
for run in runs:
|
|
440
|
+
# If specified run details does not contain attachment_id, get it from experiment asset
|
|
441
|
+
# incresing the total runs count to keep track of runs across experiments
|
|
442
|
+
total_runs += 1
|
|
443
|
+
if not run.attachment_id:
|
|
444
|
+
for run_info in ai_experiment.runs:
|
|
445
|
+
if run_info.run_id == run.run_id:
|
|
446
|
+
run = run_info
|
|
447
|
+
break
|
|
448
|
+
|
|
449
|
+
run_data = run.to_json() if hasattr(run, "to_json") else {}
|
|
450
|
+
evaluation_assets.append(
|
|
451
|
+
self.__build_evaluation_asset(
|
|
452
|
+
experiment_id, experiment_name, run_data
|
|
453
|
+
)
|
|
454
|
+
)
|
|
455
|
+
|
|
456
|
+
if total_runs > 20 or total_runs < 2:
|
|
457
|
+
message = f"Error occurred while creating AI evaluation asset. Error: The number of runs across experiments should be minimum 2 and maximum 20."
|
|
458
|
+
self.logger.error(message)
|
|
459
|
+
raise Exception(message)
|
|
460
|
+
|
|
461
|
+
# setting the default valve of monitors, name and description
|
|
462
|
+
monitors = {"agentic_ai_quality": {
|
|
463
|
+
"parameters": {"metrics_configuration": {}}}}
|
|
464
|
+
evaluation_asset_name = f"AI Agents evaluation created at {time_str}"
|
|
465
|
+
evaluation_asset_description = "AI Agents evaluation"
|
|
466
|
+
# updating the monitors, name and description if provided
|
|
467
|
+
if ai_evaluation_details:
|
|
468
|
+
monitors = ai_evaluation_details.evaluation_configuration.monitors if ai_evaluation_details.evaluation_configuration.monitors != {} else monitors
|
|
469
|
+
evaluation_asset_name = ai_evaluation_details.name if ai_evaluation_details.name != "" else evaluation_asset_name
|
|
470
|
+
evaluation_asset_description = ai_evaluation_details.description if ai_evaluation_details.description != "" else evaluation_asset_description
|
|
471
|
+
|
|
472
|
+
payload = {
|
|
473
|
+
"operational_space_id": "development",
|
|
474
|
+
"name": evaluation_asset_name,
|
|
475
|
+
"description": evaluation_asset_description,
|
|
476
|
+
"evaluation_asset_type": "ai_experiment",
|
|
477
|
+
"input_data_type": "unstructured_text",
|
|
478
|
+
"evaluation_run": {
|
|
479
|
+
"monitors": monitors,
|
|
480
|
+
"evaluation_assets": evaluation_assets,
|
|
481
|
+
},
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
ai_evaluation_url = f"{self.dataplatform_url}/v1/aigov/factsheet/ai_evaluations"
|
|
485
|
+
|
|
486
|
+
response = RestUtil.request_with_retry().post(
|
|
487
|
+
ai_evaluation_url,
|
|
488
|
+
json=payload,
|
|
489
|
+
headers=self.__get_headers(),
|
|
490
|
+
params=self.container_params,
|
|
491
|
+
verify=self.verify_ssl
|
|
492
|
+
)
|
|
493
|
+
|
|
494
|
+
if not response.ok:
|
|
495
|
+
message = f"Error occurred while creating AI Evaluation asset. Status code: {response.status_code}, Error: {response.text}"
|
|
496
|
+
self.logger.error(message)
|
|
497
|
+
raise Exception(message)
|
|
498
|
+
|
|
499
|
+
response_json = response.json()
|
|
500
|
+
ai_evaluation_asset_id = get(response_json, "asset_id")
|
|
501
|
+
|
|
502
|
+
ai_evaluation_asset = self.__map_post_response_to_ai_evaluation_asset(
|
|
503
|
+
response_json)
|
|
504
|
+
|
|
505
|
+
print(f"Created AI Evaluation asset with id {ai_evaluation_asset_id}.")
|
|
506
|
+
|
|
507
|
+
return ai_evaluation_asset
|
|
508
|
+
|
|
509
|
+
def get_ai_evaluation_asset(self, ai_evaluation_asset_id: str) -> AIEvaluationAsset:
|
|
510
|
+
"""
|
|
511
|
+
Return an instance of the AIEvaluation with the given id.
|
|
512
|
+
|
|
513
|
+
Args:
|
|
514
|
+
- ai_evaluation_asset_id: The asset id of the AI Evaluation asset.
|
|
515
|
+
Return:
|
|
516
|
+
An instance of AIEvaluationAsset with the given asset id.
|
|
517
|
+
"""
|
|
518
|
+
|
|
519
|
+
ai_evaluation_asset_url = f"{self.dataplatform_url}/v1/aigov/factsheet/ai_evaluations/{ai_evaluation_asset_id}"
|
|
520
|
+
|
|
521
|
+
response = RestUtil.request_with_retry().get(
|
|
522
|
+
ai_evaluation_asset_url,
|
|
523
|
+
headers=self.__get_headers(),
|
|
524
|
+
params=self.container_params,
|
|
525
|
+
verify=self.verify_ssl
|
|
526
|
+
)
|
|
527
|
+
|
|
528
|
+
if not response.ok:
|
|
529
|
+
message = f"Error occurred while retrieving AI Evaluation asset. Status code: {response.status_code}, Error: {response.text}"
|
|
530
|
+
self.logger.error(message)
|
|
531
|
+
raise Exception(message)
|
|
532
|
+
|
|
533
|
+
response_json = response.json()
|
|
534
|
+
|
|
535
|
+
ai_evaluation_asset = AIEvaluationAsset(**response_json)
|
|
536
|
+
|
|
537
|
+
print(
|
|
538
|
+
f"Retrieved AI Evaluation asset with id {ai_evaluation_asset_id}.\n")
|
|
539
|
+
return ai_evaluation_asset
|
|
540
|
+
|
|
541
|
+
def get_ai_evaluation_asset_href(self, ai_evaluation_asset: AIEvaluationAsset) -> str:
|
|
542
|
+
"""
|
|
543
|
+
Returns the URL of Evaluation studio UI for the given AI evaluation asset.
|
|
544
|
+
|
|
545
|
+
Args:
|
|
546
|
+
- ai_evaluation_asset: The AI Evaluation asset details.
|
|
547
|
+
Return:
|
|
548
|
+
URL of Evaluation studio UI
|
|
549
|
+
"""
|
|
550
|
+
ai_evaluation_asset_href = f"{self.dataplatform_url.replace('api.', '')}/aiopenscale/studioPage?"\
|
|
551
|
+
f"container_type={ai_evaluation_asset.container_type}&container_id={ai_evaluation_asset.container_id}&asset_id={ai_evaluation_asset.asset_id}&tearsheet_mode=true"
|
|
552
|
+
|
|
553
|
+
print(
|
|
554
|
+
f"AI Evaluation can be viewed in Evaluation Studio UI at URL: {ai_evaluation_asset_href}\n")
|
|
555
|
+
return ai_evaluation_asset_href
|
|
556
|
+
|
|
557
|
+
def list_experiments(self) -> List[AIExperiment]:
|
|
558
|
+
"""
|
|
559
|
+
List all AI Experiments under selected project.
|
|
560
|
+
|
|
561
|
+
Returns: List of AIExperiment instances.
|
|
562
|
+
"""
|
|
563
|
+
# Search using asset search API
|
|
564
|
+
search_url = f"{self.dataplatform_url}/v2/asset_types/ai_experiment/search"
|
|
565
|
+
|
|
566
|
+
search_payload = {"query": "*:*"}
|
|
567
|
+
|
|
568
|
+
response = RestUtil.request_with_retry().post(
|
|
569
|
+
search_url, json=search_payload,
|
|
570
|
+
headers=self.__get_headers(),
|
|
571
|
+
params=self.container_params,
|
|
572
|
+
verify=self.verify_ssl
|
|
573
|
+
)
|
|
574
|
+
|
|
575
|
+
if not response.ok:
|
|
576
|
+
message = f"Error occurred while searching AI experiments for project_id {self.project_id}. Status code: {response.status_code}, Error: {response.text}"
|
|
577
|
+
self.logger.error(message)
|
|
578
|
+
raise Exception(message)
|
|
579
|
+
|
|
580
|
+
response_json = response.json()
|
|
581
|
+
total_ai_experiments = get(response_json, "total_rows")
|
|
582
|
+
|
|
583
|
+
result = []
|
|
584
|
+
if total_ai_experiments > 0:
|
|
585
|
+
experiments = get(response_json, "results")
|
|
586
|
+
for experiment in experiments:
|
|
587
|
+
result.append(self.__map_ai_experiment(experiment))
|
|
588
|
+
print(
|
|
589
|
+
f"Found {total_ai_experiments} AI Experiment in project with id {self.project_id}.\n")
|
|
590
|
+
return result
|
|
591
|
+
|
|
592
|
+
def list_experiment_runs(self, ai_experiment_id) -> List[AIExperimentRun]:
|
|
593
|
+
"""
|
|
594
|
+
List all ai_experiment_runs for a given ai_experiment_id in a project.
|
|
595
|
+
|
|
596
|
+
Args:
|
|
597
|
+
-ai_experiment_id: The ID of ai experiment asset.
|
|
598
|
+
Return: List of AIExperimentRun instances.
|
|
599
|
+
"""
|
|
600
|
+
|
|
601
|
+
ai_experiment = self.get(ai_experiment_id=ai_experiment_id)
|
|
602
|
+
|
|
603
|
+
ai_experiment_runs = ai_experiment.runs
|
|
604
|
+
|
|
605
|
+
print(
|
|
606
|
+
f"Found {len(ai_experiment_runs)} runs for given ai_experiment with id {ai_experiment_id}.\n")
|
|
607
|
+
return ai_experiment_runs
|
|
608
|
+
|
|
609
|
+
def __build_evaluation_asset(
|
|
610
|
+
self, experiment_id: str, experiment_name: str, run_data: dict
|
|
611
|
+
) -> dict:
|
|
612
|
+
"""
|
|
613
|
+
Helper method to construct a single evaluation asset dictionary.
|
|
614
|
+
|
|
615
|
+
Args:
|
|
616
|
+
experiment_id (str): The ID of the experiment.
|
|
617
|
+
experiment_name (str): The name of the experiment.
|
|
618
|
+
run_data (dict): Serialized data from the experiment run.
|
|
619
|
+
|
|
620
|
+
Returns:
|
|
621
|
+
dict: A complete evaluation asset dictionary.
|
|
622
|
+
"""
|
|
623
|
+
return {
|
|
624
|
+
"id": experiment_id,
|
|
625
|
+
"name": experiment_name,
|
|
626
|
+
"container_type": self.container_type,
|
|
627
|
+
"container_id": self.container_id,
|
|
628
|
+
**run_data,
|
|
629
|
+
}
|
|
630
|
+
|
|
631
|
+
def __store_experiment_run_result(
|
|
632
|
+
self,
|
|
633
|
+
ai_experiment_id: str,
|
|
634
|
+
experiment_run_details: AIExperimentRun,
|
|
635
|
+
evaluation_result,
|
|
636
|
+
) -> dict:
|
|
637
|
+
"""
|
|
638
|
+
Stores evaluation result for an experiment run
|
|
639
|
+
1. Create attachment for storing the evaluation result
|
|
640
|
+
2. Update AI experiment with run details and corresponding attachment_id
|
|
641
|
+
Args:
|
|
642
|
+
- ai_experiment_id: The ID of AI experiment for which run result is to be stored
|
|
643
|
+
- experiment_run_details: An instance of AIExperimentRun, payload to create attachment
|
|
644
|
+
- evaluation_result: The content of attachment to be uploaded as file
|
|
645
|
+
Returns: The attachment details.
|
|
646
|
+
"""
|
|
647
|
+
run_id = experiment_run_details.run_id
|
|
648
|
+
run_name = experiment_run_details.run_name
|
|
649
|
+
|
|
650
|
+
print(
|
|
651
|
+
f"\nStoring evaluation result for experiment run {run_id} of AI experiment {ai_experiment_id}.\n")
|
|
652
|
+
|
|
653
|
+
# Creating attachment for AI experiment asset to store run result
|
|
654
|
+
attachment_payload = {
|
|
655
|
+
"asset_type": "ai_experiment",
|
|
656
|
+
"name": f"AI experiment run result for {run_name}",
|
|
657
|
+
"description": f"AI experiment run result for {run_name}",
|
|
658
|
+
"mime": "json/txt",
|
|
659
|
+
"user_data": {},
|
|
660
|
+
}
|
|
661
|
+
attachment_details = self.__create_asset_attachment(
|
|
662
|
+
ai_experiment_id, attachment_payload, evaluation_result
|
|
663
|
+
)
|
|
664
|
+
attachment_id = attachment_details.get("attachment_id")
|
|
665
|
+
|
|
666
|
+
return attachment_id
|
|
667
|
+
|
|
668
|
+
def __create_asset_attachment(
|
|
669
|
+
self, asset_id: str, attachment_payload: dict, attachment_content
|
|
670
|
+
) -> dict:
|
|
671
|
+
"""
|
|
672
|
+
Creates asset attachment for specified asset as a file containing the attachment_content
|
|
673
|
+
|
|
674
|
+
Args:
|
|
675
|
+
- asset_id: The ID of asset for which attachment is to be created
|
|
676
|
+
- attachment_payload: The payload to create attachment
|
|
677
|
+
- attachment_content: The content of attachment to be uploaded as file
|
|
678
|
+
Returns: The attachment details.
|
|
679
|
+
"""
|
|
680
|
+
|
|
681
|
+
start_time = time.time()
|
|
682
|
+
print(f"Creating attachment for asset {asset_id}.\n")
|
|
683
|
+
|
|
684
|
+
# Building the attachment URL
|
|
685
|
+
attachments_url = f"{self.dataplatform_url}/v2/assets/{asset_id}/attachments"
|
|
686
|
+
|
|
687
|
+
# Creating the attachment
|
|
688
|
+
response = RestUtil.request_with_retry().post(
|
|
689
|
+
attachments_url,
|
|
690
|
+
json=attachment_payload,
|
|
691
|
+
headers=self.__get_headers(),
|
|
692
|
+
params=self.container_params,
|
|
693
|
+
verify=self.verify_ssl
|
|
694
|
+
)
|
|
695
|
+
|
|
696
|
+
if not response.ok:
|
|
697
|
+
message = f"Failed to create attachment for asset {asset_id}. Status code: {response.status_code}, Error: {response.text}"
|
|
698
|
+
self.logger.error(message)
|
|
699
|
+
raise Exception(message)
|
|
700
|
+
|
|
701
|
+
attachment_details = json.loads(response.text)
|
|
702
|
+
|
|
703
|
+
# Fetch the attachment_id and upload URL from the details
|
|
704
|
+
attachment_id = attachment_details.get("attachment_id")
|
|
705
|
+
upload_url = attachment_details.get("url1")
|
|
706
|
+
|
|
707
|
+
# Upload the attachment content as a file
|
|
708
|
+
# In case of CPD, the attachment is uploaded to asset_files WI #https://github.ibm.com/aiopenscale/tracker/issues/51158
|
|
709
|
+
if self.api_client.is_cpd:
|
|
710
|
+
upload_url = f"{self.dataplatform_url}/{upload_url}"
|
|
711
|
+
file_path = ""
|
|
712
|
+
try:
|
|
713
|
+
with open("metrics_result.json", 'w') as file:
|
|
714
|
+
file.write(json.dumps(attachment_content))
|
|
715
|
+
file_path = file.name
|
|
716
|
+
file_stream = open(file_path, 'r')
|
|
717
|
+
files = {"file": ("metrics_result.json",
|
|
718
|
+
file_stream, "application/json")}
|
|
719
|
+
|
|
720
|
+
headers = self.__get_headers()
|
|
721
|
+
if headers.get("Content-Type"):
|
|
722
|
+
del headers["Content-Type"]
|
|
723
|
+
|
|
724
|
+
response = RestUtil.request_with_retry().put(
|
|
725
|
+
upload_url,
|
|
726
|
+
files=files,
|
|
727
|
+
headers=headers,
|
|
728
|
+
verify=self.verify_ssl
|
|
729
|
+
)
|
|
730
|
+
if not response.ok:
|
|
731
|
+
message = f"Failed to upload attachment content for asset {asset_id}. Status code: {response.status_code}, Error: {response.text}"
|
|
732
|
+
self.logger.error(message)
|
|
733
|
+
raise Exception(message)
|
|
734
|
+
except Exception as e:
|
|
735
|
+
self.logger.error(str(e))
|
|
736
|
+
raise
|
|
737
|
+
finally:
|
|
738
|
+
# Deleting the temporary file created for run result
|
|
739
|
+
try:
|
|
740
|
+
if os.path.isfile(file_path):
|
|
741
|
+
os.remove(file_path)
|
|
742
|
+
except Exception as e:
|
|
743
|
+
self.logger.error(
|
|
744
|
+
f"An error occurred while deleting the attachment file. Error: {str(e)}")
|
|
745
|
+
|
|
746
|
+
else:
|
|
747
|
+
content_bytes = json.dumps(attachment_content).encode("utf-8")
|
|
748
|
+
response = RestUtil.request_with_retry().put(
|
|
749
|
+
upload_url,
|
|
750
|
+
data=content_bytes,
|
|
751
|
+
headers=self.__get_headers(),
|
|
752
|
+
params=self.container_params,
|
|
753
|
+
verify=self.verify_ssl
|
|
754
|
+
)
|
|
755
|
+
|
|
756
|
+
if not response.ok:
|
|
757
|
+
message = f"Failed to upload attachment content for asset {asset_id}. Status code: {response.status_code}, Error: {response.text}"
|
|
758
|
+
self.logger.error(message)
|
|
759
|
+
raise Exception(message)
|
|
760
|
+
|
|
761
|
+
# Marking attachment as transfer complete
|
|
762
|
+
attachment_url = f"{attachments_url}/{attachment_id}/complete"
|
|
763
|
+
response = RestUtil.request_with_retry().post(
|
|
764
|
+
attachment_url,
|
|
765
|
+
json={},
|
|
766
|
+
headers=self.__get_headers(),
|
|
767
|
+
params=self.container_params,
|
|
768
|
+
verify=self.verify_ssl
|
|
769
|
+
)
|
|
770
|
+
|
|
771
|
+
if not response.ok:
|
|
772
|
+
message = f"Failed to mark attachment as transfer-complete for asset {asset_id}. Status code: {response.status_code}, Error: {response.text}"
|
|
773
|
+
self.logger.error(message)
|
|
774
|
+
raise Exception(message)
|
|
775
|
+
print(
|
|
776
|
+
f"Successfully created attachment {attachment_id} for asset {asset_id}. Time taken: {time.time() - start_time}.\n")
|
|
777
|
+
|
|
778
|
+
return attachment_details
|
|
779
|
+
|
|
780
|
+
def __get_headers(self) -> dict:
|
|
781
|
+
"""
|
|
782
|
+
This method will create the headers with the iam access token.
|
|
783
|
+
"""
|
|
784
|
+
|
|
785
|
+
headers = {}
|
|
786
|
+
headers["Authorization"] = f"Bearer {get_authenticator_token(self.api_client.wos_client.authenticator)}"
|
|
787
|
+
headers["Content-Type"] = "application/json"
|
|
788
|
+
headers["Accept"] = "application/json"
|
|
789
|
+
return headers
|
|
790
|
+
|
|
791
|
+
def __container_checks(self, project_id, space_id) -> None:
|
|
792
|
+
"""This method check the container type and update the container_id.
|
|
793
|
+
|
|
794
|
+
Args
|
|
795
|
+
- project_id: The project id.
|
|
796
|
+
- space_id: The space_id.
|
|
797
|
+
"""
|
|
798
|
+
if project_id and space_id:
|
|
799
|
+
message = "Both 'project_id' and 'space_id' cannot be passed together."
|
|
800
|
+
self.logger.error(message)
|
|
801
|
+
raise Exception(message)
|
|
802
|
+
|
|
803
|
+
self.container_type, self.container_id = (
|
|
804
|
+
("project", project_id) if project_id else ("space", space_id)
|
|
805
|
+
)
|
|
806
|
+
self.container_params = {
|
|
807
|
+
f"{self.container_type}_id": self.container_id}
|
|
808
|
+
|
|
809
|
+
def __map_post_response_to_ai_evaluation_asset(self, post_response: Dict) -> AIEvaluationAsset:
|
|
810
|
+
"""
|
|
811
|
+
This method is for mapping the response of post ai_evaluation call to AIEvluationAsset entity.
|
|
812
|
+
Args:
|
|
813
|
+
- post_response: The response of POST '/v1/aigov/factsheet/ai_evaluations' call.
|
|
814
|
+
Returns:
|
|
815
|
+
An Instance of AIEvaluationAsset.
|
|
816
|
+
"""
|
|
817
|
+
|
|
818
|
+
metadata = get(post_response, "metadata", {})
|
|
819
|
+
ai_evaluation = get(post_response, "entity.ai_evaluation", {})
|
|
820
|
+
evaluation_run = get(ai_evaluation, "evaluation_run", {})
|
|
821
|
+
|
|
822
|
+
return AIEvaluationAsset(
|
|
823
|
+
container_id=get(
|
|
824
|
+
metadata, f"{self.container_type}_id", default=""),
|
|
825
|
+
container_type=self.container_type,
|
|
826
|
+
container_name=get(metadata, "name", ""),
|
|
827
|
+
name=get(metadata, "name", ""),
|
|
828
|
+
description=get(metadata, "description", ""),
|
|
829
|
+
asset_type=get(metadata, "asset_type", "ai_evaluation"),
|
|
830
|
+
created_at=get(metadata, "created_at", ""),
|
|
831
|
+
owner_id=get(metadata, "owner_id", ""),
|
|
832
|
+
asset_id=get(metadata, "asset_id", ""),
|
|
833
|
+
creator_id=get(metadata, "creator_id", ""),
|
|
834
|
+
asset_details={
|
|
835
|
+
"task_ids": get(ai_evaluation, "task_ids", []),
|
|
836
|
+
"operational_space_id": get(ai_evaluation, "operational_space_id", ""),
|
|
837
|
+
"input_data_type": get(ai_evaluation, "input_data_type", ""),
|
|
838
|
+
"evaluation_asset_type": get(ai_evaluation, "evaluation_asset_type", ""),
|
|
839
|
+
},
|
|
840
|
+
evaluation_configuration=EvaluationConfig(
|
|
841
|
+
monitors=get(evaluation_run, "monitors", {}),
|
|
842
|
+
evaluation_assets=get(evaluation_run, "evaluation_assets", []),
|
|
843
|
+
),
|
|
844
|
+
href=get(post_response, "href", ""),
|
|
845
|
+
)
|
|
846
|
+
|
|
847
|
+
def __map_ai_experiment(self, experiment) -> AIExperiment:
|
|
848
|
+
"""
|
|
849
|
+
This method is for mapping the experiment json to AIExperiment entity.
|
|
850
|
+
Args:
|
|
851
|
+
- experiment: ai_experiment details in json.
|
|
852
|
+
Return: An instance of AIExperiment
|
|
853
|
+
"""
|
|
854
|
+
metadata = get(experiment, "metadata", default={})
|
|
855
|
+
ai_experiment_data = get(
|
|
856
|
+
experiment, "entity.ai_experiment", default={})
|
|
857
|
+
|
|
858
|
+
ai_experiment_details = {
|
|
859
|
+
"container_id": get(metadata, f"{self.container_type}_id", default=""),
|
|
860
|
+
"container_type": self.container_type,
|
|
861
|
+
"name": get(metadata, "name", default=""),
|
|
862
|
+
"description": get(metadata, "description", default=""),
|
|
863
|
+
"asset_type": get(metadata, "asset_type", default=""),
|
|
864
|
+
"created_at": get(metadata, "created_at", default=""),
|
|
865
|
+
"owner_id": get(metadata, "owner_id", default=""),
|
|
866
|
+
"asset_id": get(metadata, "asset_id", default=""),
|
|
867
|
+
"creator_id": get(metadata, "creator_id", default=""),
|
|
868
|
+
"component_id": get(ai_experiment_data, "component_id", default=""),
|
|
869
|
+
"component_type": get(ai_experiment_data, "component_type", default=""),
|
|
870
|
+
"component_name": get(ai_experiment_data, "component_name", default=""),
|
|
871
|
+
"runs": get(ai_experiment_data, "runs", default=[]),
|
|
872
|
+
}
|
|
873
|
+
|
|
874
|
+
ai_experiment = AIExperiment(**ai_experiment_details)
|
|
875
|
+
|
|
876
|
+
return ai_experiment
|
|
877
|
+
|
|
878
|
+
@staticmethod
|
|
879
|
+
def __process_notebook(file_path: Path):
|
|
880
|
+
"""
|
|
881
|
+
Reads the notebook from the specified path and removes all output cells before saving it as an attachment.
|
|
882
|
+
"""
|
|
883
|
+
with open(file_path, "r", encoding="utf-8") as f:
|
|
884
|
+
nb = nbformat.read(f, as_version=4)
|
|
885
|
+
|
|
886
|
+
# Process each cell in the notebook
|
|
887
|
+
for cell in nb.cells:
|
|
888
|
+
if cell.cell_type == "code":
|
|
889
|
+
# Clear all execution outputs, execution count and execution-related metadata if it exists
|
|
890
|
+
cell.outputs = []
|
|
891
|
+
cell.pop("execution_count", None)
|
|
892
|
+
cell.metadata.pop("execution", None)
|
|
893
|
+
|
|
894
|
+
return json.loads(nbformat.writes(nb))
|
|
895
|
+
|
|
896
|
+
def get_experiment_notebook(self, ai_experiment_id: str, run_id: str, custom_filename: Optional[str] = None) -> str:
|
|
897
|
+
"""
|
|
898
|
+
Download an experiment notebook from a specific AI experiment run.
|
|
899
|
+
|
|
900
|
+
Args:
|
|
901
|
+
ai_experiment_id (str): The unique identifier for the AI experiment
|
|
902
|
+
run_id (str): The specific run ID within the experiment
|
|
903
|
+
custom_filename (Optional[str]): Custom filename for the downloaded file.
|
|
904
|
+
If None, uses format: "{ai_experiment.source_name}"
|
|
905
|
+
|
|
906
|
+
Example:
|
|
907
|
+
>>> client.get_experiment_asset("exp_123", "run_456")
|
|
908
|
+
Downloaded: my_notebook.ipynb
|
|
909
|
+
"""
|
|
910
|
+
|
|
911
|
+
ai_experiment = self.get(ai_experiment_id)
|
|
912
|
+
run_detail = next(
|
|
913
|
+
(run for run in ai_experiment.runs if run.run_id == run_id),
|
|
914
|
+
None
|
|
915
|
+
)
|
|
916
|
+
if run_detail is None:
|
|
917
|
+
error_msg = f"Run '{run_id}' not found in experiment '{ai_experiment_id}'"
|
|
918
|
+
self.logger.error(error_msg)
|
|
919
|
+
raise Exception(error_msg)
|
|
920
|
+
|
|
921
|
+
attachment_id = run_detail.properties.get("notebook_attachment_id")
|
|
922
|
+
if not attachment_id:
|
|
923
|
+
error_msg = f"No notebook found for run '{run_id}' in experiment '{ai_experiment_id}'"
|
|
924
|
+
self.logger.error(error_msg)
|
|
925
|
+
raise Exception(error_msg)
|
|
926
|
+
|
|
927
|
+
attachment_details = self.__get_attachment_details(
|
|
928
|
+
ai_experiment_id, attachment_id)
|
|
929
|
+
|
|
930
|
+
s3_url = attachment_details.get("url")
|
|
931
|
+
filename = custom_filename or f"{run_detail.source_name}"
|
|
932
|
+
|
|
933
|
+
self.__download_file_from_s3(s3_url, filename, attachment_id)
|
|
934
|
+
|
|
935
|
+
print(f"Successfully downloaded experiment notebook as '{filename}'")
|
|
936
|
+
|
|
937
|
+
def __get_attachment_details(self, ai_experiment_id: str, attachment_id: str) -> Dict[str, Any]:
|
|
938
|
+
"""
|
|
939
|
+
Retrieve attachment details from the data platform API.
|
|
940
|
+
"""
|
|
941
|
+
attachments_url = f"{self.dataplatform_url}/v2/assets/{ai_experiment_id}/attachments/{attachment_id}"
|
|
942
|
+
|
|
943
|
+
response = RestUtil.request_with_retry().get(
|
|
944
|
+
attachments_url,
|
|
945
|
+
headers=self.__get_headers(),
|
|
946
|
+
params=self.container_params,
|
|
947
|
+
verify=self.verify_ssl
|
|
948
|
+
)
|
|
949
|
+
if not response.ok:
|
|
950
|
+
error_msg = (
|
|
951
|
+
f"Failed to retrieve attachment '{attachment_id}' for experiment '{ai_experiment_id}'. "
|
|
952
|
+
f"Status: {response.status_code}, Error: {response.text}"
|
|
953
|
+
)
|
|
954
|
+
self.logger.error(error_msg)
|
|
955
|
+
raise Exception(error_msg)
|
|
956
|
+
|
|
957
|
+
return json.loads(response.text)
|
|
958
|
+
|
|
959
|
+
def __download_file_from_s3(self, s3_url: str, filename: str, attachment_id: str) -> None:
|
|
960
|
+
"""
|
|
961
|
+
Download file from S3 URL and save locally.
|
|
962
|
+
"""
|
|
963
|
+
try:
|
|
964
|
+
response = requests.get(s3_url, stream=True, timeout=30)
|
|
965
|
+
if not response.ok:
|
|
966
|
+
error_msg = (
|
|
967
|
+
f"Failed to download notebook attachment '{attachment_id}'. "
|
|
968
|
+
f"Status: {response.status_code}, Error: {response.text}"
|
|
969
|
+
)
|
|
970
|
+
self.logger.error(error_msg)
|
|
971
|
+
raise Exception(error_msg)
|
|
972
|
+
|
|
973
|
+
with open(filename, "wb") as file:
|
|
974
|
+
for chunk in response.iter_content(chunk_size=8192):
|
|
975
|
+
if chunk:
|
|
976
|
+
file.write(chunk)
|
|
977
|
+
except requests.RequestException as e:
|
|
978
|
+
error_msg = f"Error downloading file: {str(e)}"
|
|
979
|
+
self.logger.error(error_msg)
|
|
980
|
+
raise Exception(error_msg)
|