ibm-watsonx-gov 1.3.3__cp313-cp313-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ibm_watsonx_gov/__init__.py +8 -0
- ibm_watsonx_gov/agent_catalog/__init__.py +8 -0
- ibm_watsonx_gov/agent_catalog/clients/__init__.py +14 -0
- ibm_watsonx_gov/agent_catalog/clients/ai_agent_client.py +333 -0
- ibm_watsonx_gov/agent_catalog/core/__init__.py +8 -0
- ibm_watsonx_gov/agent_catalog/core/agent_loader.py +202 -0
- ibm_watsonx_gov/agent_catalog/core/agents.py +134 -0
- ibm_watsonx_gov/agent_catalog/entities/__init__.py +8 -0
- ibm_watsonx_gov/agent_catalog/entities/ai_agent.py +599 -0
- ibm_watsonx_gov/agent_catalog/utils/__init__.py +8 -0
- ibm_watsonx_gov/agent_catalog/utils/constants.py +36 -0
- ibm_watsonx_gov/agent_catalog/utils/notebook_utils.py +70 -0
- ibm_watsonx_gov/ai_experiments/__init__.py +8 -0
- ibm_watsonx_gov/ai_experiments/ai_experiments_client.py +980 -0
- ibm_watsonx_gov/ai_experiments/utils/__init__.py +8 -0
- ibm_watsonx_gov/ai_experiments/utils/ai_experiment_utils.py +139 -0
- ibm_watsonx_gov/clients/__init__.py +0 -0
- ibm_watsonx_gov/clients/api_client.py +99 -0
- ibm_watsonx_gov/clients/segment_client.py +46 -0
- ibm_watsonx_gov/clients/usage_client.cp313-win_amd64.pyd +0 -0
- ibm_watsonx_gov/clients/wx_ai_client.py +87 -0
- ibm_watsonx_gov/config/__init__.py +14 -0
- ibm_watsonx_gov/config/agentic_ai_configuration.py +225 -0
- ibm_watsonx_gov/config/gen_ai_configuration.py +129 -0
- ibm_watsonx_gov/config/model_risk_configuration.py +173 -0
- ibm_watsonx_gov/config/predictive_ai_configuration.py +20 -0
- ibm_watsonx_gov/entities/__init__.py +8 -0
- ibm_watsonx_gov/entities/agentic_app.py +209 -0
- ibm_watsonx_gov/entities/agentic_evaluation_result.py +185 -0
- ibm_watsonx_gov/entities/ai_evaluation.py +290 -0
- ibm_watsonx_gov/entities/ai_experiment.py +419 -0
- ibm_watsonx_gov/entities/base_classes.py +134 -0
- ibm_watsonx_gov/entities/container.py +54 -0
- ibm_watsonx_gov/entities/credentials.py +633 -0
- ibm_watsonx_gov/entities/criteria.py +508 -0
- ibm_watsonx_gov/entities/enums.py +274 -0
- ibm_watsonx_gov/entities/evaluation_result.py +444 -0
- ibm_watsonx_gov/entities/foundation_model.py +490 -0
- ibm_watsonx_gov/entities/llm_judge.py +44 -0
- ibm_watsonx_gov/entities/locale.py +17 -0
- ibm_watsonx_gov/entities/mapping.py +49 -0
- ibm_watsonx_gov/entities/metric.py +211 -0
- ibm_watsonx_gov/entities/metric_threshold.py +36 -0
- ibm_watsonx_gov/entities/model_provider.py +329 -0
- ibm_watsonx_gov/entities/model_risk_result.py +43 -0
- ibm_watsonx_gov/entities/monitor.py +71 -0
- ibm_watsonx_gov/entities/prompt_setup.py +40 -0
- ibm_watsonx_gov/entities/state.py +22 -0
- ibm_watsonx_gov/entities/utils.py +99 -0
- ibm_watsonx_gov/evaluators/__init__.py +26 -0
- ibm_watsonx_gov/evaluators/agentic_evaluator.py +2725 -0
- ibm_watsonx_gov/evaluators/agentic_traces_evaluator.py +115 -0
- ibm_watsonx_gov/evaluators/base_evaluator.py +22 -0
- ibm_watsonx_gov/evaluators/impl/__init__.py +0 -0
- ibm_watsonx_gov/evaluators/impl/evaluate_metrics_impl.cp313-win_amd64.pyd +0 -0
- ibm_watsonx_gov/evaluators/impl/evaluate_model_risk_impl.cp313-win_amd64.pyd +0 -0
- ibm_watsonx_gov/evaluators/metrics_evaluator.py +187 -0
- ibm_watsonx_gov/evaluators/model_risk_evaluator.py +89 -0
- ibm_watsonx_gov/evaluators/traces_evaluator.py +93 -0
- ibm_watsonx_gov/metric_groups/answer_quality/answer_quality_decorator.py +66 -0
- ibm_watsonx_gov/metric_groups/content_safety/content_safety_decorator.py +76 -0
- ibm_watsonx_gov/metric_groups/readability/readability_decorator.py +59 -0
- ibm_watsonx_gov/metric_groups/retrieval_quality/retrieval_quality_decorator.py +63 -0
- ibm_watsonx_gov/metric_groups/usage/usage_decorator.py +58 -0
- ibm_watsonx_gov/metrics/__init__.py +74 -0
- ibm_watsonx_gov/metrics/answer_relevance/__init__.py +8 -0
- ibm_watsonx_gov/metrics/answer_relevance/answer_relevance_decorator.py +63 -0
- ibm_watsonx_gov/metrics/answer_relevance/answer_relevance_metric.py +260 -0
- ibm_watsonx_gov/metrics/answer_similarity/__init__.py +0 -0
- ibm_watsonx_gov/metrics/answer_similarity/answer_similarity_decorator.py +66 -0
- ibm_watsonx_gov/metrics/answer_similarity/answer_similarity_metric.py +219 -0
- ibm_watsonx_gov/metrics/average_precision/__init__.py +0 -0
- ibm_watsonx_gov/metrics/average_precision/average_precision_decorator.py +62 -0
- ibm_watsonx_gov/metrics/average_precision/average_precision_metric.py +174 -0
- ibm_watsonx_gov/metrics/base_metric_decorator.py +193 -0
- ibm_watsonx_gov/metrics/context_relevance/__init__.py +8 -0
- ibm_watsonx_gov/metrics/context_relevance/context_relevance_decorator.py +60 -0
- ibm_watsonx_gov/metrics/context_relevance/context_relevance_metric.py +414 -0
- ibm_watsonx_gov/metrics/cost/__init__.py +8 -0
- ibm_watsonx_gov/metrics/cost/cost_decorator.py +58 -0
- ibm_watsonx_gov/metrics/cost/cost_metric.py +155 -0
- ibm_watsonx_gov/metrics/duration/__init__.py +8 -0
- ibm_watsonx_gov/metrics/duration/duration_decorator.py +59 -0
- ibm_watsonx_gov/metrics/duration/duration_metric.py +111 -0
- ibm_watsonx_gov/metrics/evasiveness/__init__.py +8 -0
- ibm_watsonx_gov/metrics/evasiveness/evasiveness_decorator.py +61 -0
- ibm_watsonx_gov/metrics/evasiveness/evasiveness_metric.py +103 -0
- ibm_watsonx_gov/metrics/faithfulness/__init__.py +8 -0
- ibm_watsonx_gov/metrics/faithfulness/faithfulness_decorator.py +65 -0
- ibm_watsonx_gov/metrics/faithfulness/faithfulness_metric.py +254 -0
- ibm_watsonx_gov/metrics/hap/__init__.py +16 -0
- ibm_watsonx_gov/metrics/hap/hap_decorator.py +58 -0
- ibm_watsonx_gov/metrics/hap/hap_metric.py +98 -0
- ibm_watsonx_gov/metrics/hap/input_hap_metric.py +104 -0
- ibm_watsonx_gov/metrics/hap/output_hap_metric.py +110 -0
- ibm_watsonx_gov/metrics/harm/__init__.py +8 -0
- ibm_watsonx_gov/metrics/harm/harm_decorator.py +60 -0
- ibm_watsonx_gov/metrics/harm/harm_metric.py +103 -0
- ibm_watsonx_gov/metrics/harm_engagement/__init__.py +8 -0
- ibm_watsonx_gov/metrics/harm_engagement/harm_engagement_decorator.py +61 -0
- ibm_watsonx_gov/metrics/harm_engagement/harm_engagement_metric.py +103 -0
- ibm_watsonx_gov/metrics/hit_rate/__init__.py +0 -0
- ibm_watsonx_gov/metrics/hit_rate/hit_rate_decorator.py +59 -0
- ibm_watsonx_gov/metrics/hit_rate/hit_rate_metric.py +167 -0
- ibm_watsonx_gov/metrics/input_token_count/__init__.py +8 -0
- ibm_watsonx_gov/metrics/input_token_count/input_token_count_decorator.py +58 -0
- ibm_watsonx_gov/metrics/input_token_count/input_token_count_metric.py +112 -0
- ibm_watsonx_gov/metrics/jailbreak/__init__.py +8 -0
- ibm_watsonx_gov/metrics/jailbreak/jailbreak_decorator.py +60 -0
- ibm_watsonx_gov/metrics/jailbreak/jailbreak_metric.py +103 -0
- ibm_watsonx_gov/metrics/keyword_detection/keyword_detection_decorator.py +58 -0
- ibm_watsonx_gov/metrics/keyword_detection/keyword_detection_metric.py +111 -0
- ibm_watsonx_gov/metrics/llm_validation/__init__.py +8 -0
- ibm_watsonx_gov/metrics/llm_validation/evaluation_criteria.py +84 -0
- ibm_watsonx_gov/metrics/llm_validation/llm_validation_constants.py +24 -0
- ibm_watsonx_gov/metrics/llm_validation/llm_validation_decorator.py +54 -0
- ibm_watsonx_gov/metrics/llm_validation/llm_validation_impl.py +525 -0
- ibm_watsonx_gov/metrics/llm_validation/llm_validation_metric.py +258 -0
- ibm_watsonx_gov/metrics/llm_validation/llm_validation_prompts.py +106 -0
- ibm_watsonx_gov/metrics/llmaj/__init__.py +0 -0
- ibm_watsonx_gov/metrics/llmaj/llmaj_metric.py +298 -0
- ibm_watsonx_gov/metrics/ndcg/__init__.py +0 -0
- ibm_watsonx_gov/metrics/ndcg/ndcg_decorator.py +61 -0
- ibm_watsonx_gov/metrics/ndcg/ndcg_metric.py +166 -0
- ibm_watsonx_gov/metrics/output_token_count/__init__.py +8 -0
- ibm_watsonx_gov/metrics/output_token_count/output_token_count_decorator.py +58 -0
- ibm_watsonx_gov/metrics/output_token_count/output_token_count_metric.py +112 -0
- ibm_watsonx_gov/metrics/pii/__init__.py +16 -0
- ibm_watsonx_gov/metrics/pii/input_pii_metric.py +102 -0
- ibm_watsonx_gov/metrics/pii/output_pii_metric.py +107 -0
- ibm_watsonx_gov/metrics/pii/pii_decorator.py +59 -0
- ibm_watsonx_gov/metrics/pii/pii_metric.py +96 -0
- ibm_watsonx_gov/metrics/profanity/__init__.py +8 -0
- ibm_watsonx_gov/metrics/profanity/profanity_decorator.py +60 -0
- ibm_watsonx_gov/metrics/profanity/profanity_metric.py +103 -0
- ibm_watsonx_gov/metrics/prompt_safety_risk/__init__.py +8 -0
- ibm_watsonx_gov/metrics/prompt_safety_risk/prompt_safety_risk_decorator.py +57 -0
- ibm_watsonx_gov/metrics/prompt_safety_risk/prompt_safety_risk_metric.py +128 -0
- ibm_watsonx_gov/metrics/reciprocal_rank/__init__.py +0 -0
- ibm_watsonx_gov/metrics/reciprocal_rank/reciprocal_rank_decorator.py +62 -0
- ibm_watsonx_gov/metrics/reciprocal_rank/reciprocal_rank_metric.py +162 -0
- ibm_watsonx_gov/metrics/regex_detection/regex_detection_decorator.py +58 -0
- ibm_watsonx_gov/metrics/regex_detection/regex_detection_metric.py +106 -0
- ibm_watsonx_gov/metrics/retrieval_precision/__init__.py +0 -0
- ibm_watsonx_gov/metrics/retrieval_precision/retrieval_precision_decorator.py +62 -0
- ibm_watsonx_gov/metrics/retrieval_precision/retrieval_precision_metric.py +170 -0
- ibm_watsonx_gov/metrics/sexual_content/__init__.py +8 -0
- ibm_watsonx_gov/metrics/sexual_content/sexual_content_decorator.py +61 -0
- ibm_watsonx_gov/metrics/sexual_content/sexual_content_metric.py +103 -0
- ibm_watsonx_gov/metrics/social_bias/__init__.py +8 -0
- ibm_watsonx_gov/metrics/social_bias/social_bias_decorator.py +62 -0
- ibm_watsonx_gov/metrics/social_bias/social_bias_metric.py +103 -0
- ibm_watsonx_gov/metrics/status/__init__.py +0 -0
- ibm_watsonx_gov/metrics/status/status_metric.py +113 -0
- ibm_watsonx_gov/metrics/text_grade_level/__init__.py +8 -0
- ibm_watsonx_gov/metrics/text_grade_level/text_grade_level_decorator.py +59 -0
- ibm_watsonx_gov/metrics/text_grade_level/text_grade_level_metric.py +127 -0
- ibm_watsonx_gov/metrics/text_reading_ease/__init__.py +8 -0
- ibm_watsonx_gov/metrics/text_reading_ease/text_reading_ease_decorator.py +59 -0
- ibm_watsonx_gov/metrics/text_reading_ease/text_reading_ease_metric.py +123 -0
- ibm_watsonx_gov/metrics/tool_call_accuracy/__init__.py +0 -0
- ibm_watsonx_gov/metrics/tool_call_accuracy/tool_call_accuracy_decorator.py +67 -0
- ibm_watsonx_gov/metrics/tool_call_accuracy/tool_call_accuracy_metric.py +162 -0
- ibm_watsonx_gov/metrics/tool_call_parameter_accuracy/__init__.py +0 -0
- ibm_watsonx_gov/metrics/tool_call_parameter_accuracy/tool_call_parameter_accuracy_decorator.py +68 -0
- ibm_watsonx_gov/metrics/tool_call_parameter_accuracy/tool_call_parameter_accuracy_metric.py +151 -0
- ibm_watsonx_gov/metrics/tool_call_relevance/__init__.py +0 -0
- ibm_watsonx_gov/metrics/tool_call_relevance/tool_call_relevance_decorator.py +71 -0
- ibm_watsonx_gov/metrics/tool_call_relevance/tool_call_relevance_metric.py +166 -0
- ibm_watsonx_gov/metrics/tool_call_syntactic_accuracy/__init__.py +0 -0
- ibm_watsonx_gov/metrics/tool_call_syntactic_accuracy/tool_call_syntactic_accuracy_decorator.py +66 -0
- ibm_watsonx_gov/metrics/tool_call_syntactic_accuracy/tool_call_syntactic_accuracy_metric.py +121 -0
- ibm_watsonx_gov/metrics/topic_relevance/__init__.py +8 -0
- ibm_watsonx_gov/metrics/topic_relevance/topic_relevance_decorator.py +57 -0
- ibm_watsonx_gov/metrics/topic_relevance/topic_relevance_metric.py +106 -0
- ibm_watsonx_gov/metrics/unethical_behavior/__init__.py +8 -0
- ibm_watsonx_gov/metrics/unethical_behavior/unethical_behavior_decorator.py +61 -0
- ibm_watsonx_gov/metrics/unethical_behavior/unethical_behavior_metric.py +103 -0
- ibm_watsonx_gov/metrics/unsuccessful_requests/__init__.py +0 -0
- ibm_watsonx_gov/metrics/unsuccessful_requests/unsuccessful_requests_decorator.py +66 -0
- ibm_watsonx_gov/metrics/unsuccessful_requests/unsuccessful_requests_metric.py +128 -0
- ibm_watsonx_gov/metrics/user_id/__init__.py +0 -0
- ibm_watsonx_gov/metrics/user_id/user_id_metric.py +111 -0
- ibm_watsonx_gov/metrics/utils.py +440 -0
- ibm_watsonx_gov/metrics/violence/__init__.py +8 -0
- ibm_watsonx_gov/metrics/violence/violence_decorator.py +60 -0
- ibm_watsonx_gov/metrics/violence/violence_metric.py +103 -0
- ibm_watsonx_gov/prompt_evaluator/__init__.py +9 -0
- ibm_watsonx_gov/prompt_evaluator/impl/__init__.py +8 -0
- ibm_watsonx_gov/prompt_evaluator/impl/prompt_evaluator_impl.py +554 -0
- ibm_watsonx_gov/prompt_evaluator/impl/pta_lifecycle_evaluator.py +2332 -0
- ibm_watsonx_gov/prompt_evaluator/prompt_evaluator.py +262 -0
- ibm_watsonx_gov/providers/__init__.py +8 -0
- ibm_watsonx_gov/providers/detectors_provider.cp313-win_amd64.pyd +0 -0
- ibm_watsonx_gov/providers/detectors_provider.py +415 -0
- ibm_watsonx_gov/providers/eval_assist_provider.cp313-win_amd64.pyd +0 -0
- ibm_watsonx_gov/providers/eval_assist_provider.py +266 -0
- ibm_watsonx_gov/providers/inference_engines/__init__.py +0 -0
- ibm_watsonx_gov/providers/inference_engines/custom_inference_engine.py +165 -0
- ibm_watsonx_gov/providers/inference_engines/portkey_inference_engine.py +57 -0
- ibm_watsonx_gov/providers/llmevalkit/__init__.py +0 -0
- ibm_watsonx_gov/providers/llmevalkit/ciso_agent/main.py +516 -0
- ibm_watsonx_gov/providers/llmevalkit/ciso_agent/preprocess_log.py +111 -0
- ibm_watsonx_gov/providers/llmevalkit/ciso_agent/utils.py +186 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/README.md +411 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/__init__.py +27 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/comparison/README.md +306 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/comparison/__init__.py +89 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/comparison/comparators/__init__.py +30 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/comparison/comparators/base.py +411 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/comparison/comparators/code_agent.py +1254 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/comparison/comparators/exact_match.py +134 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/comparison/comparators/fuzzy_string.py +104 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/comparison/comparators/hybrid.py +516 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/comparison/comparators/llm_judge.py +1882 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/comparison/pipeline.py +387 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/comparison/types.py +178 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/comparison/utils.py +298 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/consts.py +33 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/__init__.py +31 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/base.py +26 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/function_call/__init__.py +4 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/function_call/general.py +46 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/function_call/general_metrics.json +783 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/function_call/general_metrics_runtime.json +580 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/function_selection/__init__.py +6 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/function_selection/function_selection.py +28 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/function_selection/function_selection_metrics.json +599 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/function_selection/function_selection_metrics_runtime.json +477 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/loader.py +259 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/parameter/__init__.py +7 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/parameter/parameter.py +52 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/parameter/parameter_metrics.json +613 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/parameter/parameter_metrics_runtime.json +489 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/trajectory/__init__.py +7 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/trajectory/trajectory.py +43 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/trajectory/trajectory_metrics.json +161 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/pipeline/__init__.py +0 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/pipeline/adapters.py +102 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/pipeline/pipeline.py +355 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/pipeline/semantic_checker.py +816 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/pipeline/static_checker.py +297 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/pipeline/transformation_prompts.py +509 -0
- ibm_watsonx_gov/providers/llmevalkit/function_calling/pipeline/types.py +596 -0
- ibm_watsonx_gov/providers/llmevalkit/llm/README.md +375 -0
- ibm_watsonx_gov/providers/llmevalkit/llm/__init__.py +137 -0
- ibm_watsonx_gov/providers/llmevalkit/llm/base.py +426 -0
- ibm_watsonx_gov/providers/llmevalkit/llm/output_parser.py +364 -0
- ibm_watsonx_gov/providers/llmevalkit/llm/providers/__init__.py +0 -0
- ibm_watsonx_gov/providers/llmevalkit/llm/providers/consts.py +7 -0
- ibm_watsonx_gov/providers/llmevalkit/llm/providers/ibm_watsonx_ai/__init__.py +0 -0
- ibm_watsonx_gov/providers/llmevalkit/llm/providers/ibm_watsonx_ai/ibm_watsonx_ai.py +656 -0
- ibm_watsonx_gov/providers/llmevalkit/llm/providers/litellm/__init__.py +0 -0
- ibm_watsonx_gov/providers/llmevalkit/llm/providers/litellm/litellm.py +509 -0
- ibm_watsonx_gov/providers/llmevalkit/llm/providers/litellm/rits.py +224 -0
- ibm_watsonx_gov/providers/llmevalkit/llm/providers/litellm/watsonx.py +60 -0
- ibm_watsonx_gov/providers/llmevalkit/llm/providers/mock_llm_client.py +75 -0
- ibm_watsonx_gov/providers/llmevalkit/llm/providers/openai/__init__.py +0 -0
- ibm_watsonx_gov/providers/llmevalkit/llm/providers/openai/openai.py +639 -0
- ibm_watsonx_gov/providers/llmevalkit/llm/providers/wxo_ai_gateway/__init__.py +0 -0
- ibm_watsonx_gov/providers/llmevalkit/llm/providers/wxo_ai_gateway/wxo_ai_gateway.py +134 -0
- ibm_watsonx_gov/providers/llmevalkit/llm/providers/wxo_ai_gateway/wxo_ai_gateway_inference.py +214 -0
- ibm_watsonx_gov/providers/llmevalkit/llm/types.py +136 -0
- ibm_watsonx_gov/providers/llmevalkit/metrics/__init__.py +4 -0
- ibm_watsonx_gov/providers/llmevalkit/metrics/field.py +255 -0
- ibm_watsonx_gov/providers/llmevalkit/metrics/metric.py +332 -0
- ibm_watsonx_gov/providers/llmevalkit/metrics/metrics_runner.py +188 -0
- ibm_watsonx_gov/providers/llmevalkit/metrics/prompt.py +403 -0
- ibm_watsonx_gov/providers/llmevalkit/metrics/utils.py +46 -0
- ibm_watsonx_gov/providers/llmevalkit/prompt/__init__.py +0 -0
- ibm_watsonx_gov/providers/llmevalkit/prompt/runner.py +144 -0
- ibm_watsonx_gov/providers/tool_call_metric_provider.py +455 -0
- ibm_watsonx_gov/providers/unitxt_provider.cp313-win_amd64.pyd +0 -0
- ibm_watsonx_gov/tools/__init__.py +10 -0
- ibm_watsonx_gov/tools/clients/__init__.py +11 -0
- ibm_watsonx_gov/tools/clients/ai_tool_client.py +405 -0
- ibm_watsonx_gov/tools/clients/detector_client.py +82 -0
- ibm_watsonx_gov/tools/core/__init__.py +8 -0
- ibm_watsonx_gov/tools/core/tool_loader.py +237 -0
- ibm_watsonx_gov/tools/entities/__init__.py +8 -0
- ibm_watsonx_gov/tools/entities/ai_tools.py +435 -0
- ibm_watsonx_gov/tools/onboarding/create/answer_relevance_detector.json +57 -0
- ibm_watsonx_gov/tools/onboarding/create/chromadb_retrieval_tool.json +63 -0
- ibm_watsonx_gov/tools/onboarding/create/context_relevance_detector.json +57 -0
- ibm_watsonx_gov/tools/onboarding/create/duduckgo_search_tool.json +53 -0
- ibm_watsonx_gov/tools/onboarding/create/google_search_tool.json +62 -0
- ibm_watsonx_gov/tools/onboarding/create/hap_detector.json +70 -0
- ibm_watsonx_gov/tools/onboarding/create/jailbreak_detector.json +70 -0
- ibm_watsonx_gov/tools/onboarding/create/pii_detector.json +36 -0
- ibm_watsonx_gov/tools/onboarding/create/prompt_safety_risk_detector.json +69 -0
- ibm_watsonx_gov/tools/onboarding/create/topic_relevance_detector.json +57 -0
- ibm_watsonx_gov/tools/onboarding/create/weather_tool.json +39 -0
- ibm_watsonx_gov/tools/onboarding/create/webcrawler_tool.json +34 -0
- ibm_watsonx_gov/tools/onboarding/create/wikipedia_search_tool.json +53 -0
- ibm_watsonx_gov/tools/onboarding/delete/delete_tools.json +4 -0
- ibm_watsonx_gov/tools/onboarding/update/google_search_tool.json +38 -0
- ibm_watsonx_gov/tools/ootb/__init__.py +8 -0
- ibm_watsonx_gov/tools/ootb/detectors/__init__.py +8 -0
- ibm_watsonx_gov/tools/ootb/detectors/hap_detector_tool.py +109 -0
- ibm_watsonx_gov/tools/ootb/detectors/jailbreak_detector_tool.py +104 -0
- ibm_watsonx_gov/tools/ootb/detectors/pii_detector_tool.py +83 -0
- ibm_watsonx_gov/tools/ootb/detectors/prompt_safety_risk_detector_tool.py +111 -0
- ibm_watsonx_gov/tools/ootb/detectors/topic_relevance_detector_tool.py +101 -0
- ibm_watsonx_gov/tools/ootb/rag/__init__.py +8 -0
- ibm_watsonx_gov/tools/ootb/rag/answer_relevance_detector_tool.py +119 -0
- ibm_watsonx_gov/tools/ootb/rag/context_relevance_detector_tool.py +118 -0
- ibm_watsonx_gov/tools/ootb/search/__init__.py +8 -0
- ibm_watsonx_gov/tools/ootb/search/duckduckgo_search_tool.py +62 -0
- ibm_watsonx_gov/tools/ootb/search/google_search_tool.py +105 -0
- ibm_watsonx_gov/tools/ootb/search/weather_tool.py +95 -0
- ibm_watsonx_gov/tools/ootb/search/web_crawler_tool.py +69 -0
- ibm_watsonx_gov/tools/ootb/search/wikipedia_search_tool.py +63 -0
- ibm_watsonx_gov/tools/ootb/vectordb/__init__.py +8 -0
- ibm_watsonx_gov/tools/ootb/vectordb/chromadb_retriever_tool.py +111 -0
- ibm_watsonx_gov/tools/rest_api/__init__.py +10 -0
- ibm_watsonx_gov/tools/rest_api/restapi_tool.py +72 -0
- ibm_watsonx_gov/tools/schemas/__init__.py +10 -0
- ibm_watsonx_gov/tools/schemas/search_tool_schema.py +46 -0
- ibm_watsonx_gov/tools/schemas/vectordb_retrieval_schema.py +55 -0
- ibm_watsonx_gov/tools/utils/__init__.py +14 -0
- ibm_watsonx_gov/tools/utils/constants.py +69 -0
- ibm_watsonx_gov/tools/utils/display_utils.py +38 -0
- ibm_watsonx_gov/tools/utils/environment.py +108 -0
- ibm_watsonx_gov/tools/utils/package_utils.py +40 -0
- ibm_watsonx_gov/tools/utils/platform_url_mapping.cp313-win_amd64.pyd +0 -0
- ibm_watsonx_gov/tools/utils/python_utils.py +68 -0
- ibm_watsonx_gov/tools/utils/tool_utils.py +206 -0
- ibm_watsonx_gov/traces/__init__.py +8 -0
- ibm_watsonx_gov/traces/span_exporter.py +195 -0
- ibm_watsonx_gov/traces/span_node.py +251 -0
- ibm_watsonx_gov/traces/span_util.py +153 -0
- ibm_watsonx_gov/traces/trace_utils.py +1074 -0
- ibm_watsonx_gov/utils/__init__.py +8 -0
- ibm_watsonx_gov/utils/aggregation_util.py +346 -0
- ibm_watsonx_gov/utils/async_util.py +62 -0
- ibm_watsonx_gov/utils/authenticator.py +144 -0
- ibm_watsonx_gov/utils/constants.py +15 -0
- ibm_watsonx_gov/utils/errors.py +40 -0
- ibm_watsonx_gov/utils/gov_sdk_logger.py +39 -0
- ibm_watsonx_gov/utils/insights_generator.py +1285 -0
- ibm_watsonx_gov/utils/python_utils.py +425 -0
- ibm_watsonx_gov/utils/rest_util.py +73 -0
- ibm_watsonx_gov/utils/segment_batch_manager.py +162 -0
- ibm_watsonx_gov/utils/singleton_meta.py +25 -0
- ibm_watsonx_gov/utils/url_mapping.cp313-win_amd64.pyd +0 -0
- ibm_watsonx_gov/utils/validation_util.py +126 -0
- ibm_watsonx_gov/visualizations/__init__.py +13 -0
- ibm_watsonx_gov/visualizations/metric_descriptions.py +57 -0
- ibm_watsonx_gov/visualizations/model_insights.py +1304 -0
- ibm_watsonx_gov/visualizations/visualization_utils.py +75 -0
- ibm_watsonx_gov-1.3.3.dist-info/METADATA +93 -0
- ibm_watsonx_gov-1.3.3.dist-info/RECORD +353 -0
- ibm_watsonx_gov-1.3.3.dist-info/WHEEL +4 -0
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
{
|
|
2
|
+
"display_name": "Wikipedia Search",
|
|
3
|
+
"tool_name": "wikipedia_search_tool",
|
|
4
|
+
"service_provider_type": "IBM",
|
|
5
|
+
"description": "Search Wikipedia and return results for given query",
|
|
6
|
+
"inventory_id": "<attribute_value>",
|
|
7
|
+
"reusable": "true",
|
|
8
|
+
"category": ["Search"],
|
|
9
|
+
"framework": ["langchain","langgraph"],
|
|
10
|
+
"code": {
|
|
11
|
+
"source_code_base64": "",
|
|
12
|
+
"run_time_details": {
|
|
13
|
+
"engine": "Python >=3.11"
|
|
14
|
+
}
|
|
15
|
+
},
|
|
16
|
+
"config":{
|
|
17
|
+
"description": "Configuration schema for OOTB search tools",
|
|
18
|
+
"properties": {
|
|
19
|
+
"top_k_results": {
|
|
20
|
+
"default": 3,
|
|
21
|
+
"description": "Number of search results to retrieve",
|
|
22
|
+
"title": "Top K Results",
|
|
23
|
+
"type": "integer"
|
|
24
|
+
}
|
|
25
|
+
},
|
|
26
|
+
"title": "SearchToolConfig",
|
|
27
|
+
"type": "object"
|
|
28
|
+
},
|
|
29
|
+
"schema":{
|
|
30
|
+
"properties": {
|
|
31
|
+
"query": {
|
|
32
|
+
"description": "Search the input from the given search tool",
|
|
33
|
+
"title": "Query",
|
|
34
|
+
"type": "string"
|
|
35
|
+
},
|
|
36
|
+
"top_k_results": {
|
|
37
|
+
"default": 3,
|
|
38
|
+
"description": "Number of search results to retrieve",
|
|
39
|
+
"title": "Top K Results",
|
|
40
|
+
"type": "integer"
|
|
41
|
+
}
|
|
42
|
+
},
|
|
43
|
+
"required": [
|
|
44
|
+
"query"
|
|
45
|
+
],
|
|
46
|
+
"title": "SearchToolInput",
|
|
47
|
+
"type": "object"
|
|
48
|
+
},
|
|
49
|
+
"dependencies": {
|
|
50
|
+
"remote_services":[],
|
|
51
|
+
"run_time_packages":["wikipedia"]
|
|
52
|
+
}
|
|
53
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
{
|
|
2
|
+
"tool_name": "google_search_tool",
|
|
3
|
+
"version": "1.1.0",
|
|
4
|
+
"patch_tool": [
|
|
5
|
+
{
|
|
6
|
+
"op": "replace",
|
|
7
|
+
"path": "/config",
|
|
8
|
+
"value": {
|
|
9
|
+
"description": "Configuration schema for OOTB search tools",
|
|
10
|
+
"properties": {
|
|
11
|
+
"top_k_results": {
|
|
12
|
+
"default": 3,
|
|
13
|
+
"description": "Number of search results to retrieve",
|
|
14
|
+
"title": "Top K Results",
|
|
15
|
+
"type": "integer"
|
|
16
|
+
},
|
|
17
|
+
"serpapi_key": {
|
|
18
|
+
"description": "API key used to authenticate requests to SerpAPI. You can generate your API key from https://serpapi.com/manage-api-key.",
|
|
19
|
+
"title": "SerpAPI Key",
|
|
20
|
+
"type": "string"
|
|
21
|
+
}
|
|
22
|
+
},
|
|
23
|
+
"title": "SearchToolConfig",
|
|
24
|
+
"type": "object",
|
|
25
|
+
"required": [
|
|
26
|
+
"serpapi_key"
|
|
27
|
+
]
|
|
28
|
+
}
|
|
29
|
+
},
|
|
30
|
+
{
|
|
31
|
+
"op": "replace",
|
|
32
|
+
"path": "/dependencies/run_time_packages",
|
|
33
|
+
"value": [
|
|
34
|
+
"google-search-results"
|
|
35
|
+
]
|
|
36
|
+
}
|
|
37
|
+
]
|
|
38
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
# ----------------------------------------------------------------------------------------------------
|
|
2
|
+
# IBM Confidential
|
|
3
|
+
# Licensed Materials - Property of IBM
|
|
4
|
+
# 5737-H76, 5900-A3Q
|
|
5
|
+
# © Copyright IBM Corp. 2025 All Rights Reserved.
|
|
6
|
+
# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
|
|
7
|
+
# GSA ADPSchedule Contract with IBM Corp.
|
|
8
|
+
# ----------------------------------------------------------------------------------------------------
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
# ----------------------------------------------------------------------------------------------------
|
|
2
|
+
# IBM Confidential
|
|
3
|
+
# Licensed Materials - Property of IBM
|
|
4
|
+
# 5737-H76, 5900-A3Q
|
|
5
|
+
# © Copyright IBM Corp. 2025 All Rights Reserved.
|
|
6
|
+
# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
|
|
7
|
+
# GSA ADPSchedule Contract with IBM Corp.
|
|
8
|
+
# ----------------------------------------------------------------------------------------------------
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
# ----------------------------------------------------------------------------------------------------
|
|
2
|
+
# IBM Confidential
|
|
3
|
+
# Licensed Materials - Property of IBM
|
|
4
|
+
# 5737-H76, 5900-A3Q
|
|
5
|
+
# © Copyright IBM Corp. 2025 All Rights Reserved.
|
|
6
|
+
# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
|
|
7
|
+
# GSA ADPSchedule Contract with IBM Corp.
|
|
8
|
+
# ----------------------------------------------------------------------------------------------------
|
|
9
|
+
|
|
10
|
+
import ibm_watsonx_gov.tools.clients.detector_client as dc
|
|
11
|
+
|
|
12
|
+
from typing import Annotated, List, Optional, Type
|
|
13
|
+
from langchain.callbacks.manager import CallbackManagerForToolRun
|
|
14
|
+
from langchain.tools import BaseTool
|
|
15
|
+
from pydantic import BaseModel, Field, PrivateAttr
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class HAPInput(BaseModel):
|
|
21
|
+
"""
|
|
22
|
+
Model that can be used for setting input args for HAPDetectorTool
|
|
23
|
+
"""
|
|
24
|
+
input: Annotated[str,
|
|
25
|
+
Field(..., description="Detect HAP for the given input")]
|
|
26
|
+
threshold: Annotated[Optional[float], Field(
|
|
27
|
+
0.8, description="Threshold for hap detection in the input, values range from 0.0 to 1.0")]
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class HAPConfig(BaseModel):
|
|
31
|
+
"""
|
|
32
|
+
Model that can be used for setting threshold for HAPDetectorTool
|
|
33
|
+
"""
|
|
34
|
+
threshold: Annotated[Optional[float], Field(
|
|
35
|
+
0.8, description="Threshold for hap detection in the input, values range from 0.0 to 1.0")]
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class HAPDetectorTool(BaseTool):
|
|
39
|
+
"""
|
|
40
|
+
Tool to detect Hate, Abuse and Profanity content in input string
|
|
41
|
+
|
|
42
|
+
Examples:
|
|
43
|
+
Basic usage
|
|
44
|
+
.. code-block:: python
|
|
45
|
+
|
|
46
|
+
hap_detector_tool = HAPDetectorTool()
|
|
47
|
+
hap_detector_tool.invoke({"input":"<USER_INPUT>"})
|
|
48
|
+
"""
|
|
49
|
+
name: str = "hap_detector"
|
|
50
|
+
description: str = "Tool to detect Hate, Abuse and Profanity content in the input string"
|
|
51
|
+
args_schema: Type[BaseModel] = HAPInput
|
|
52
|
+
|
|
53
|
+
_url: any = PrivateAttr()
|
|
54
|
+
_threshold: any = PrivateAttr()
|
|
55
|
+
|
|
56
|
+
def __init__(self, **kwargs):
|
|
57
|
+
super().__init__(**kwargs)
|
|
58
|
+
config = HAPConfig(**kwargs)
|
|
59
|
+
self._threshold = config.threshold
|
|
60
|
+
self._url = dc.DETECTIONS_URL.format(dc.get_base_url())
|
|
61
|
+
self._base_payload = dc.get_base_payload(dc.HAP,detector_params= {
|
|
62
|
+
"threshold": 0.0
|
|
63
|
+
})
|
|
64
|
+
|
|
65
|
+
# Define HAP Tool
|
|
66
|
+
def _run(self,
|
|
67
|
+
input: str,
|
|
68
|
+
run_manager: Optional[CallbackManagerForToolRun] = None,
|
|
69
|
+
**kwargs) -> List[str]:
|
|
70
|
+
"""
|
|
71
|
+
Detects Hate, Abuse and Profanity content in the input string
|
|
72
|
+
|
|
73
|
+
Sample Response:
|
|
74
|
+
{
|
|
75
|
+
'detections': [{'start': 0,
|
|
76
|
+
'end': 46,
|
|
77
|
+
'text': '<USER_INPUT>',
|
|
78
|
+
'detection_type': 'hap',
|
|
79
|
+
'detection': 'has_HAP',
|
|
80
|
+
'score': 0.9495730400085448,
|
|
81
|
+
}],
|
|
82
|
+
'is_hap_detected': True
|
|
83
|
+
}
|
|
84
|
+
"""
|
|
85
|
+
|
|
86
|
+
threshold = kwargs.get("threshold",self._threshold)
|
|
87
|
+
# By default threshold is 0.0 and score is compared with the default/user provided threshold
|
|
88
|
+
|
|
89
|
+
payload = dc.get_payload(self._base_payload,detector_payload={
|
|
90
|
+
"input":input
|
|
91
|
+
})
|
|
92
|
+
response = dc.call_detections(self._url, payload)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
#Add boolean to response
|
|
96
|
+
response["is_hap_detected"] = False
|
|
97
|
+
for detection in response.get("detections"):
|
|
98
|
+
if detection.get("score") > threshold:
|
|
99
|
+
response["is_hap_detected"] = True
|
|
100
|
+
break
|
|
101
|
+
|
|
102
|
+
return response
|
|
103
|
+
|
|
104
|
+
async def _arun(self,
|
|
105
|
+
input: str,
|
|
106
|
+
run_manager: Optional[CallbackManagerForToolRun] = None,
|
|
107
|
+
**kwargs) -> List[str]:
|
|
108
|
+
|
|
109
|
+
return self._run(input , kwargs)
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
# ----------------------------------------------------------------------------------------------------
|
|
2
|
+
# IBM Confidential
|
|
3
|
+
# Licensed Materials - Property of IBM
|
|
4
|
+
# 5737-H76, 5900-A3Q
|
|
5
|
+
# © Copyright IBM Corp. 2025 All Rights Reserved.
|
|
6
|
+
# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
|
|
7
|
+
# GSA ADPSchedule Contract with IBM Corp.
|
|
8
|
+
# ----------------------------------------------------------------------------------------------------
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
|
|
12
|
+
import ibm_watsonx_gov.tools.clients.detector_client as dc
|
|
13
|
+
|
|
14
|
+
from typing import Annotated, List, Optional, Type
|
|
15
|
+
|
|
16
|
+
from langchain.callbacks.manager import CallbackManagerForToolRun
|
|
17
|
+
from langchain.tools import BaseTool
|
|
18
|
+
from pydantic import BaseModel, Field, PrivateAttr
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class JailBreakInput(BaseModel):
|
|
22
|
+
"""
|
|
23
|
+
Model that can be used for setting input args for JailBreakDetectorTool
|
|
24
|
+
"""
|
|
25
|
+
input: Annotated[str,
|
|
26
|
+
Field(..., description="Detect JailBreak for the given input")]
|
|
27
|
+
threshold: Annotated[Optional[float], Field(
|
|
28
|
+
0.8, description="Threshold for detecting jailbreak in the input, values range from 0.0 to 1.0")]
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class JailBreakConfig(BaseModel):
|
|
32
|
+
"""
|
|
33
|
+
Model that can be used for setting threshold for JailBreakDetectorTool
|
|
34
|
+
"""
|
|
35
|
+
threshold: Annotated[Optional[float], Field(
|
|
36
|
+
0.8, description="Threshold for detecting jailbreak in the input, values range from 0.0 to 1.0")]
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class JailBreakDetectorTool(BaseTool):
|
|
40
|
+
"""
|
|
41
|
+
Tool to detect deliberate circumvention of AI systems built-in safeguards or ethical guidelines.
|
|
42
|
+
This involves crafting specific prompts or scenarios designed to manipulate the AI into generating
|
|
43
|
+
restricted or inappropriate content.
|
|
44
|
+
|
|
45
|
+
Examples:
|
|
46
|
+
Basic usage
|
|
47
|
+
.. code-block:: python
|
|
48
|
+
|
|
49
|
+
jailbreak_detector_tool = JailBreakDetectorTool()
|
|
50
|
+
jailbreak_detector_tool.invoke({"input":"<USER_INPUT>"})
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
name: str = "jailbreak_detector"
|
|
54
|
+
description: str = """Tool that detects deliberate circumvention of AI systems built-in safeguards or ethical guidelines.
|
|
55
|
+
This involves crafting specific prompts or scenarios designed to manipulate the AI into generating
|
|
56
|
+
restricted or inappropriate content."""
|
|
57
|
+
args_schema: Type[BaseModel] = JailBreakInput
|
|
58
|
+
|
|
59
|
+
_url: any = PrivateAttr()
|
|
60
|
+
_threshold: any = PrivateAttr()
|
|
61
|
+
|
|
62
|
+
def __init__(self, **kwargs):
|
|
63
|
+
super().__init__(**kwargs)
|
|
64
|
+
config = JailBreakConfig(**kwargs)
|
|
65
|
+
self._threshold = config.threshold
|
|
66
|
+
self._url = dc.DETECTIONS_URL.format(dc.get_base_url())
|
|
67
|
+
self._base_payload = dc.get_base_payload(dc.GRANITE_GUADIAN,detector_params= {
|
|
68
|
+
"risk_name": "jailbreak",
|
|
69
|
+
"threshold": 0.0
|
|
70
|
+
})
|
|
71
|
+
|
|
72
|
+
# Define JailBreak Tool
|
|
73
|
+
def _run(self,
|
|
74
|
+
input: str,
|
|
75
|
+
run_manager: Optional[CallbackManagerForToolRun] = None,
|
|
76
|
+
**kwargs) -> List[str]:
|
|
77
|
+
"""
|
|
78
|
+
Sample Response:
|
|
79
|
+
{'detections': [{'start': 0,
|
|
80
|
+
'end': 26,
|
|
81
|
+
'text': '<USER_INPUT>',
|
|
82
|
+
'detection_type': 'risk',
|
|
83
|
+
'detection': 'Yes',
|
|
84
|
+
'score': 0.9688562154769896}],
|
|
85
|
+
'is_jailbreak_detected': True}
|
|
86
|
+
"""
|
|
87
|
+
threshold = kwargs.get("threshold",self._threshold)
|
|
88
|
+
payload = dc.get_payload(self._base_payload,detector_payload={
|
|
89
|
+
"input":input
|
|
90
|
+
})
|
|
91
|
+
response = dc.call_detections(self._url, payload)
|
|
92
|
+
|
|
93
|
+
if response.get("detections")[0].get("score") > threshold:
|
|
94
|
+
response["is_jailbreak_detected"] = True
|
|
95
|
+
else:
|
|
96
|
+
response["is_jailbreak_detected"] = False
|
|
97
|
+
return response
|
|
98
|
+
|
|
99
|
+
async def _arun(self,
|
|
100
|
+
input: str,
|
|
101
|
+
run_manager: Optional[CallbackManagerForToolRun] = None,
|
|
102
|
+
**kwargs) -> List[str]:
|
|
103
|
+
|
|
104
|
+
return self._run(input , kwargs)
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
# ----------------------------------------------------------------------------------------------------
|
|
2
|
+
# IBM Confidential
|
|
3
|
+
# Licensed Materials - Property of IBM
|
|
4
|
+
# 5737-H76, 5900-A3Q
|
|
5
|
+
# © Copyright IBM Corp. 2025 All Rights Reserved.
|
|
6
|
+
# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
|
|
7
|
+
# GSA ADPSchedule Contract with IBM Corp.
|
|
8
|
+
# ----------------------------------------------------------------------------------------------------
|
|
9
|
+
|
|
10
|
+
import ibm_watsonx_gov.tools.clients.detector_client as dc
|
|
11
|
+
|
|
12
|
+
from typing import Annotated, List, Optional, Type
|
|
13
|
+
from langchain.callbacks.manager import CallbackManagerForToolRun
|
|
14
|
+
from langchain.tools import BaseTool
|
|
15
|
+
from pydantic import BaseModel, Field, PrivateAttr
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class PIIInput(BaseModel):
|
|
19
|
+
"""
|
|
20
|
+
Model that can be used for setting input args for PIIDetectorTool
|
|
21
|
+
"""
|
|
22
|
+
input: Annotated[str,
|
|
23
|
+
Field(..., description="Detect PII for the given input")]
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class PIIDetectorTool(BaseTool):
|
|
27
|
+
"""
|
|
28
|
+
Tool to detect PII content in input string
|
|
29
|
+
|
|
30
|
+
Examples:
|
|
31
|
+
Basic usage
|
|
32
|
+
.. code-block:: python
|
|
33
|
+
|
|
34
|
+
pii_detector_tool = PIIDetectorTool()
|
|
35
|
+
pii_detector_tool.invoke({"input":"<USER_INPUT>"})
|
|
36
|
+
"""
|
|
37
|
+
name: str = "pii_detector"
|
|
38
|
+
description: str = "Tool that detects Personally Identifiable Information(PII) information in the input string."
|
|
39
|
+
args_schema: Type[BaseModel] = PIIInput
|
|
40
|
+
_url: any = PrivateAttr()
|
|
41
|
+
|
|
42
|
+
def __init__(self, **kwargs):
|
|
43
|
+
super().__init__(**kwargs)
|
|
44
|
+
self._url = dc.DETECTIONS_URL.format(dc.get_base_url())
|
|
45
|
+
self._base_payload = dc.get_base_payload(dc.PII)
|
|
46
|
+
|
|
47
|
+
# Define PII Tool
|
|
48
|
+
def _run(self,
|
|
49
|
+
input: str,
|
|
50
|
+
run_manager: Optional[CallbackManagerForToolRun] = None,
|
|
51
|
+
**kwargs) -> List[str]:
|
|
52
|
+
|
|
53
|
+
"""
|
|
54
|
+
Sample Response:
|
|
55
|
+
{'detections': [{'start': 15,
|
|
56
|
+
'end': 30,
|
|
57
|
+
'text': 'johndoe@abc.com',
|
|
58
|
+
'detection_type': 'pii',
|
|
59
|
+
'detection': 'EmailAddress',
|
|
60
|
+
'score': 0.8}],
|
|
61
|
+
'is_pii_detected': True}
|
|
62
|
+
"""
|
|
63
|
+
|
|
64
|
+
payload = dc.get_payload(self._base_payload,detector_payload={
|
|
65
|
+
"input":input
|
|
66
|
+
})
|
|
67
|
+
response = dc.call_detections(self._url, payload)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
if response.get("detections") == []:
|
|
71
|
+
# No PII information detected
|
|
72
|
+
response["is_pii_detected"] = False
|
|
73
|
+
else:
|
|
74
|
+
response["is_pii_detected"] = True
|
|
75
|
+
|
|
76
|
+
return response
|
|
77
|
+
|
|
78
|
+
async def _arun(self,
|
|
79
|
+
input: str,
|
|
80
|
+
run_manager: Optional[CallbackManagerForToolRun] = None,
|
|
81
|
+
**kwargs) -> List[str]:
|
|
82
|
+
|
|
83
|
+
return self._run(input , kwargs)
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
# ----------------------------------------------------------------------------------------------------
|
|
2
|
+
# IBM Confidential
|
|
3
|
+
# Licensed Materials - Property of IBM
|
|
4
|
+
# 5737-H76, 5900-A3Q
|
|
5
|
+
# © Copyright IBM Corp. 2025 All Rights Reserved.
|
|
6
|
+
# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
|
|
7
|
+
# GSA ADPSchedule Contract with IBM Corp.
|
|
8
|
+
# ----------------------------------------------------------------------------------------------------
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
import ibm_watsonx_gov.tools.clients.detector_client as dc
|
|
12
|
+
from typing import Annotated, List, Optional, Type
|
|
13
|
+
from langchain.callbacks.manager import CallbackManagerForToolRun
|
|
14
|
+
from langchain.tools import BaseTool
|
|
15
|
+
from pydantic import BaseModel, Field, PrivateAttr
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class PromptSafetyRiskInput(BaseModel):
|
|
19
|
+
"""
|
|
20
|
+
Model that can be used for setting input args for PromptSafetyRiskTool
|
|
21
|
+
"""
|
|
22
|
+
input: Annotated[str,
|
|
23
|
+
Field(..., description="Detect PromptSafetyRisk for the given input")]
|
|
24
|
+
threshold: Annotated[Optional[float], Field(
|
|
25
|
+
default=0.8, description="Threshold for prompt safety risk detection in the input, values range from 0.0 to 1.0")]
|
|
26
|
+
system_prompt: Annotated[str,
|
|
27
|
+
Field(..., description="System prompt to configure the chatbot's behavior.")]
|
|
28
|
+
enable_two_level_detection: Annotated[Optional[bool], Field(
|
|
29
|
+
default=True, description="Enable or disable two level detection")]
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class PromptSafetyRiskConfig(BaseModel):
|
|
33
|
+
"""
|
|
34
|
+
Model that can be used for setting threshold for PromptSafetyRiskTool
|
|
35
|
+
"""
|
|
36
|
+
threshold: Annotated[Optional[float], Field(
|
|
37
|
+
default=0.8, description="Threshold for prompt safety risk detection in the input, values range from 0.0 to 1.0")]
|
|
38
|
+
enable_two_level_detection: Annotated[Optional[bool], Field(
|
|
39
|
+
default=False, description="Enable or disable two level detection")]
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class PromptSafetyRiskDetectorTool(BaseTool):
|
|
43
|
+
"""
|
|
44
|
+
Prompt Safety Risk tool measures the intent of jailbreak and/or prompt injection in the input sent to the LLM.
|
|
45
|
+
|
|
46
|
+
Examples:
|
|
47
|
+
Basic usage
|
|
48
|
+
.. code-block:: python
|
|
49
|
+
|
|
50
|
+
prompt_safety_detector_tool = PromptSafetyRiskTool()
|
|
51
|
+
prompt_safety_detector_tool.invoke({"input":"<USER_INPUT>", "system_prompt":"<SYSTEM_PROMPT>"})
|
|
52
|
+
"""
|
|
53
|
+
|
|
54
|
+
name: str = "prompt_safety_risk_detector"
|
|
55
|
+
description: str = "Tool that detects off topic and prompt injection attempts in user input"
|
|
56
|
+
args_schema: Type[BaseModel] = PromptSafetyRiskInput
|
|
57
|
+
|
|
58
|
+
_threshold: any = PrivateAttr()
|
|
59
|
+
_system_prompt: any = PrivateAttr()
|
|
60
|
+
_url: any = PrivateAttr()
|
|
61
|
+
_enable_two_level_detection: any = PrivateAttr()
|
|
62
|
+
|
|
63
|
+
def __init__(self, **kwargs):
|
|
64
|
+
super().__init__(**kwargs)
|
|
65
|
+
config = PromptSafetyRiskConfig(**kwargs)
|
|
66
|
+
self._threshold = config.threshold
|
|
67
|
+
self._enable_two_level_detection = config.enable_two_level_detection
|
|
68
|
+
self._url = dc.DETECTIONS_URL.format(dc.get_base_url())
|
|
69
|
+
|
|
70
|
+
def _run(self,
|
|
71
|
+
input: str,
|
|
72
|
+
system_prompt:str,
|
|
73
|
+
run_manager: Optional[CallbackManagerForToolRun] = None,
|
|
74
|
+
**kwargs) -> List[str]:
|
|
75
|
+
|
|
76
|
+
"""
|
|
77
|
+
Sample Response:
|
|
78
|
+
{'detections': [{'start': 0,
|
|
79
|
+
'end': 36,
|
|
80
|
+
'text': '<USER_INPUT>',
|
|
81
|
+
'detection_type': 'topic_relevance',
|
|
82
|
+
'detection': 'off-topic',
|
|
83
|
+
'score': 0.9422797560691832}],
|
|
84
|
+
'is_prompt_safety_risk_detected': True}
|
|
85
|
+
"""
|
|
86
|
+
|
|
87
|
+
threshold = threshold = kwargs.get("threshold",self._threshold)
|
|
88
|
+
enable_two_level_detection = kwargs.get("enable_two_level_detection",self._enable_two_level_detection)
|
|
89
|
+
|
|
90
|
+
# Default or user provided threshold and enable_two_level_detection is passed to the endpoint
|
|
91
|
+
detector_params = {
|
|
92
|
+
"system_prompt": system_prompt,
|
|
93
|
+
"enable_two_level_detection": enable_two_level_detection,
|
|
94
|
+
"threshold": 0.0
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
base_payload = dc.get_base_payload(dc.PROMPT_SAFETY_RISK,detector_params)
|
|
98
|
+
|
|
99
|
+
# payload = get_payload(
|
|
100
|
+
# self.detector_name, input, detector_params=detector_params)
|
|
101
|
+
payload = dc.get_payload(base_payload,detector_payload={
|
|
102
|
+
"input":input
|
|
103
|
+
})
|
|
104
|
+
response = dc.call_detections(self._url, payload)
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
if len(response.get("detections")) > 0 and response.get("detections")[0].get("score") > threshold:
|
|
108
|
+
response["is_prompt_safety_risk_detected"] = True
|
|
109
|
+
else:
|
|
110
|
+
response["is_prompt_safety_risk_detected"] = False
|
|
111
|
+
return response
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
# ----------------------------------------------------------------------------------------------------
|
|
2
|
+
# IBM Confidential
|
|
3
|
+
# Licensed Materials - Property of IBM
|
|
4
|
+
# 5737-H76, 5900-A3Q
|
|
5
|
+
# © Copyright IBM Corp. 2025 All Rights Reserved.
|
|
6
|
+
# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
|
|
7
|
+
# GSA ADPSchedule Contract with IBM Corp.
|
|
8
|
+
# ----------------------------------------------------------------------------------------------------
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
import ibm_watsonx_gov.tools.clients.detector_client as dc
|
|
12
|
+
|
|
13
|
+
from typing import Annotated, List, Optional, Type
|
|
14
|
+
|
|
15
|
+
from langchain.callbacks.manager import CallbackManagerForToolRun
|
|
16
|
+
from langchain.tools import BaseTool
|
|
17
|
+
from pydantic import BaseModel, Field, PrivateAttr
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class TopicRelevanceDetectorInput(BaseModel):
|
|
21
|
+
"""
|
|
22
|
+
Model that can be used for setting input args for TopicRelevanceDetectorTool
|
|
23
|
+
"""
|
|
24
|
+
input: Annotated[str,
|
|
25
|
+
Field(..., description="Detect topic relevance for the given input")]
|
|
26
|
+
threshold: Annotated[Optional[float], Field(
|
|
27
|
+
default=0.8, description="Threshold for off-topic detection in the input, values range from 0.0 to 1.0")]
|
|
28
|
+
system_prompt: Annotated[str,
|
|
29
|
+
Field(..., description="System prompt to configure the chatbot's behavior.")]
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class TopicRelevanceDetectorConfig(BaseModel):
|
|
33
|
+
"""
|
|
34
|
+
Model that can be used for setting threshold for TopicRelevanceDetectorTool
|
|
35
|
+
"""
|
|
36
|
+
threshold: Annotated[Optional[float], Field(
|
|
37
|
+
default=0.8, description="Threshold for off-topic detection in the input, values range from 0.0 to 1.0")]
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class TopicRelevanceDetectorTool(BaseTool):
|
|
41
|
+
"""
|
|
42
|
+
Tool that analyzes whether the user input is relevant to the topic defined in the system prompt and measures the degree of alignment.
|
|
43
|
+
|
|
44
|
+
Examples:
|
|
45
|
+
Basic usage
|
|
46
|
+
.. code-block:: python
|
|
47
|
+
|
|
48
|
+
topic_relevance_detector = TopicRelevanceDetectorTool()
|
|
49
|
+
topic_relevance_detector.invoke({"input":"<USER_INPUT>", "system_prompt":"<SYSTEM_PROMPT>"})
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
name: str = "topic_relevance_detector"
|
|
53
|
+
description: str = "Tool that analyzes whether the user input is relevant to the topic defined in the system prompt and measures the degree of alignment."
|
|
54
|
+
args_schema: Type[BaseModel] = TopicRelevanceDetectorInput
|
|
55
|
+
|
|
56
|
+
_threshold: any = PrivateAttr()
|
|
57
|
+
_system_prompt: any = PrivateAttr()
|
|
58
|
+
_url: any = PrivateAttr()
|
|
59
|
+
|
|
60
|
+
def __init__(self, **kwargs):
|
|
61
|
+
super().__init__(**kwargs)
|
|
62
|
+
config = TopicRelevanceDetectorConfig(**kwargs)
|
|
63
|
+
self._threshold = config.threshold
|
|
64
|
+
self._url = dc.DETECTIONS_URL.format(dc.get_base_url())
|
|
65
|
+
|
|
66
|
+
def _run(self,
|
|
67
|
+
input: str,
|
|
68
|
+
system_prompt,
|
|
69
|
+
run_manager: Optional[CallbackManagerForToolRun] = None,
|
|
70
|
+
**kwargs) -> List[str]:
|
|
71
|
+
"""
|
|
72
|
+
Sample Response:
|
|
73
|
+
{'detections': [{'start': 0,
|
|
74
|
+
'end': 35,
|
|
75
|
+
'text': '<USER_INPUT>',
|
|
76
|
+
'detection_type': 'topic_relevance',
|
|
77
|
+
'detection': 'off-topic',
|
|
78
|
+
'score': 0.847662091255188}],
|
|
79
|
+
'is_topic_relevant': False}
|
|
80
|
+
"""
|
|
81
|
+
|
|
82
|
+
threshold = kwargs.get("threshold",self._threshold)
|
|
83
|
+
# By default threshold is 0.0 and score is compared with the default/user provided threshold
|
|
84
|
+
detector_params = {
|
|
85
|
+
"system_prompt": system_prompt,
|
|
86
|
+
"threshold": 0.0
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
base_payload = dc.get_base_payload(dc.TOPIC_RELEVANCE,detector_params)
|
|
90
|
+
payload = dc.get_payload(base_payload,detector_payload={
|
|
91
|
+
"input":input
|
|
92
|
+
})
|
|
93
|
+
response = dc.call_detections(self._url, payload)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
#Propogate boolean values
|
|
97
|
+
if len(response.get("detections")) > 0 and response.get("detections")[0].get("score") > threshold:
|
|
98
|
+
response["is_topic_relevant"] = False
|
|
99
|
+
else:
|
|
100
|
+
response["is_topic_relevant"] = True
|
|
101
|
+
return response
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
# ----------------------------------------------------------------------------------------------------
|
|
2
|
+
# IBM Confidential
|
|
3
|
+
# Licensed Materials - Property of IBM
|
|
4
|
+
# 5737-H76, 5900-A3Q
|
|
5
|
+
# © Copyright IBM Corp. 2025 All Rights Reserved.
|
|
6
|
+
# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
|
|
7
|
+
# GSA ADPSchedule Contract with IBM Corp.
|
|
8
|
+
# ----------------------------------------------------------------------------------------------------
|