ibm-watsonx-gov 1.3.3__cp313-cp313-win_amd64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (353) hide show
  1. ibm_watsonx_gov/__init__.py +8 -0
  2. ibm_watsonx_gov/agent_catalog/__init__.py +8 -0
  3. ibm_watsonx_gov/agent_catalog/clients/__init__.py +14 -0
  4. ibm_watsonx_gov/agent_catalog/clients/ai_agent_client.py +333 -0
  5. ibm_watsonx_gov/agent_catalog/core/__init__.py +8 -0
  6. ibm_watsonx_gov/agent_catalog/core/agent_loader.py +202 -0
  7. ibm_watsonx_gov/agent_catalog/core/agents.py +134 -0
  8. ibm_watsonx_gov/agent_catalog/entities/__init__.py +8 -0
  9. ibm_watsonx_gov/agent_catalog/entities/ai_agent.py +599 -0
  10. ibm_watsonx_gov/agent_catalog/utils/__init__.py +8 -0
  11. ibm_watsonx_gov/agent_catalog/utils/constants.py +36 -0
  12. ibm_watsonx_gov/agent_catalog/utils/notebook_utils.py +70 -0
  13. ibm_watsonx_gov/ai_experiments/__init__.py +8 -0
  14. ibm_watsonx_gov/ai_experiments/ai_experiments_client.py +980 -0
  15. ibm_watsonx_gov/ai_experiments/utils/__init__.py +8 -0
  16. ibm_watsonx_gov/ai_experiments/utils/ai_experiment_utils.py +139 -0
  17. ibm_watsonx_gov/clients/__init__.py +0 -0
  18. ibm_watsonx_gov/clients/api_client.py +99 -0
  19. ibm_watsonx_gov/clients/segment_client.py +46 -0
  20. ibm_watsonx_gov/clients/usage_client.cp313-win_amd64.pyd +0 -0
  21. ibm_watsonx_gov/clients/wx_ai_client.py +87 -0
  22. ibm_watsonx_gov/config/__init__.py +14 -0
  23. ibm_watsonx_gov/config/agentic_ai_configuration.py +225 -0
  24. ibm_watsonx_gov/config/gen_ai_configuration.py +129 -0
  25. ibm_watsonx_gov/config/model_risk_configuration.py +173 -0
  26. ibm_watsonx_gov/config/predictive_ai_configuration.py +20 -0
  27. ibm_watsonx_gov/entities/__init__.py +8 -0
  28. ibm_watsonx_gov/entities/agentic_app.py +209 -0
  29. ibm_watsonx_gov/entities/agentic_evaluation_result.py +185 -0
  30. ibm_watsonx_gov/entities/ai_evaluation.py +290 -0
  31. ibm_watsonx_gov/entities/ai_experiment.py +419 -0
  32. ibm_watsonx_gov/entities/base_classes.py +134 -0
  33. ibm_watsonx_gov/entities/container.py +54 -0
  34. ibm_watsonx_gov/entities/credentials.py +633 -0
  35. ibm_watsonx_gov/entities/criteria.py +508 -0
  36. ibm_watsonx_gov/entities/enums.py +274 -0
  37. ibm_watsonx_gov/entities/evaluation_result.py +444 -0
  38. ibm_watsonx_gov/entities/foundation_model.py +490 -0
  39. ibm_watsonx_gov/entities/llm_judge.py +44 -0
  40. ibm_watsonx_gov/entities/locale.py +17 -0
  41. ibm_watsonx_gov/entities/mapping.py +49 -0
  42. ibm_watsonx_gov/entities/metric.py +211 -0
  43. ibm_watsonx_gov/entities/metric_threshold.py +36 -0
  44. ibm_watsonx_gov/entities/model_provider.py +329 -0
  45. ibm_watsonx_gov/entities/model_risk_result.py +43 -0
  46. ibm_watsonx_gov/entities/monitor.py +71 -0
  47. ibm_watsonx_gov/entities/prompt_setup.py +40 -0
  48. ibm_watsonx_gov/entities/state.py +22 -0
  49. ibm_watsonx_gov/entities/utils.py +99 -0
  50. ibm_watsonx_gov/evaluators/__init__.py +26 -0
  51. ibm_watsonx_gov/evaluators/agentic_evaluator.py +2725 -0
  52. ibm_watsonx_gov/evaluators/agentic_traces_evaluator.py +115 -0
  53. ibm_watsonx_gov/evaluators/base_evaluator.py +22 -0
  54. ibm_watsonx_gov/evaluators/impl/__init__.py +0 -0
  55. ibm_watsonx_gov/evaluators/impl/evaluate_metrics_impl.cp313-win_amd64.pyd +0 -0
  56. ibm_watsonx_gov/evaluators/impl/evaluate_model_risk_impl.cp313-win_amd64.pyd +0 -0
  57. ibm_watsonx_gov/evaluators/metrics_evaluator.py +187 -0
  58. ibm_watsonx_gov/evaluators/model_risk_evaluator.py +89 -0
  59. ibm_watsonx_gov/evaluators/traces_evaluator.py +93 -0
  60. ibm_watsonx_gov/metric_groups/answer_quality/answer_quality_decorator.py +66 -0
  61. ibm_watsonx_gov/metric_groups/content_safety/content_safety_decorator.py +76 -0
  62. ibm_watsonx_gov/metric_groups/readability/readability_decorator.py +59 -0
  63. ibm_watsonx_gov/metric_groups/retrieval_quality/retrieval_quality_decorator.py +63 -0
  64. ibm_watsonx_gov/metric_groups/usage/usage_decorator.py +58 -0
  65. ibm_watsonx_gov/metrics/__init__.py +74 -0
  66. ibm_watsonx_gov/metrics/answer_relevance/__init__.py +8 -0
  67. ibm_watsonx_gov/metrics/answer_relevance/answer_relevance_decorator.py +63 -0
  68. ibm_watsonx_gov/metrics/answer_relevance/answer_relevance_metric.py +260 -0
  69. ibm_watsonx_gov/metrics/answer_similarity/__init__.py +0 -0
  70. ibm_watsonx_gov/metrics/answer_similarity/answer_similarity_decorator.py +66 -0
  71. ibm_watsonx_gov/metrics/answer_similarity/answer_similarity_metric.py +219 -0
  72. ibm_watsonx_gov/metrics/average_precision/__init__.py +0 -0
  73. ibm_watsonx_gov/metrics/average_precision/average_precision_decorator.py +62 -0
  74. ibm_watsonx_gov/metrics/average_precision/average_precision_metric.py +174 -0
  75. ibm_watsonx_gov/metrics/base_metric_decorator.py +193 -0
  76. ibm_watsonx_gov/metrics/context_relevance/__init__.py +8 -0
  77. ibm_watsonx_gov/metrics/context_relevance/context_relevance_decorator.py +60 -0
  78. ibm_watsonx_gov/metrics/context_relevance/context_relevance_metric.py +414 -0
  79. ibm_watsonx_gov/metrics/cost/__init__.py +8 -0
  80. ibm_watsonx_gov/metrics/cost/cost_decorator.py +58 -0
  81. ibm_watsonx_gov/metrics/cost/cost_metric.py +155 -0
  82. ibm_watsonx_gov/metrics/duration/__init__.py +8 -0
  83. ibm_watsonx_gov/metrics/duration/duration_decorator.py +59 -0
  84. ibm_watsonx_gov/metrics/duration/duration_metric.py +111 -0
  85. ibm_watsonx_gov/metrics/evasiveness/__init__.py +8 -0
  86. ibm_watsonx_gov/metrics/evasiveness/evasiveness_decorator.py +61 -0
  87. ibm_watsonx_gov/metrics/evasiveness/evasiveness_metric.py +103 -0
  88. ibm_watsonx_gov/metrics/faithfulness/__init__.py +8 -0
  89. ibm_watsonx_gov/metrics/faithfulness/faithfulness_decorator.py +65 -0
  90. ibm_watsonx_gov/metrics/faithfulness/faithfulness_metric.py +254 -0
  91. ibm_watsonx_gov/metrics/hap/__init__.py +16 -0
  92. ibm_watsonx_gov/metrics/hap/hap_decorator.py +58 -0
  93. ibm_watsonx_gov/metrics/hap/hap_metric.py +98 -0
  94. ibm_watsonx_gov/metrics/hap/input_hap_metric.py +104 -0
  95. ibm_watsonx_gov/metrics/hap/output_hap_metric.py +110 -0
  96. ibm_watsonx_gov/metrics/harm/__init__.py +8 -0
  97. ibm_watsonx_gov/metrics/harm/harm_decorator.py +60 -0
  98. ibm_watsonx_gov/metrics/harm/harm_metric.py +103 -0
  99. ibm_watsonx_gov/metrics/harm_engagement/__init__.py +8 -0
  100. ibm_watsonx_gov/metrics/harm_engagement/harm_engagement_decorator.py +61 -0
  101. ibm_watsonx_gov/metrics/harm_engagement/harm_engagement_metric.py +103 -0
  102. ibm_watsonx_gov/metrics/hit_rate/__init__.py +0 -0
  103. ibm_watsonx_gov/metrics/hit_rate/hit_rate_decorator.py +59 -0
  104. ibm_watsonx_gov/metrics/hit_rate/hit_rate_metric.py +167 -0
  105. ibm_watsonx_gov/metrics/input_token_count/__init__.py +8 -0
  106. ibm_watsonx_gov/metrics/input_token_count/input_token_count_decorator.py +58 -0
  107. ibm_watsonx_gov/metrics/input_token_count/input_token_count_metric.py +112 -0
  108. ibm_watsonx_gov/metrics/jailbreak/__init__.py +8 -0
  109. ibm_watsonx_gov/metrics/jailbreak/jailbreak_decorator.py +60 -0
  110. ibm_watsonx_gov/metrics/jailbreak/jailbreak_metric.py +103 -0
  111. ibm_watsonx_gov/metrics/keyword_detection/keyword_detection_decorator.py +58 -0
  112. ibm_watsonx_gov/metrics/keyword_detection/keyword_detection_metric.py +111 -0
  113. ibm_watsonx_gov/metrics/llm_validation/__init__.py +8 -0
  114. ibm_watsonx_gov/metrics/llm_validation/evaluation_criteria.py +84 -0
  115. ibm_watsonx_gov/metrics/llm_validation/llm_validation_constants.py +24 -0
  116. ibm_watsonx_gov/metrics/llm_validation/llm_validation_decorator.py +54 -0
  117. ibm_watsonx_gov/metrics/llm_validation/llm_validation_impl.py +525 -0
  118. ibm_watsonx_gov/metrics/llm_validation/llm_validation_metric.py +258 -0
  119. ibm_watsonx_gov/metrics/llm_validation/llm_validation_prompts.py +106 -0
  120. ibm_watsonx_gov/metrics/llmaj/__init__.py +0 -0
  121. ibm_watsonx_gov/metrics/llmaj/llmaj_metric.py +298 -0
  122. ibm_watsonx_gov/metrics/ndcg/__init__.py +0 -0
  123. ibm_watsonx_gov/metrics/ndcg/ndcg_decorator.py +61 -0
  124. ibm_watsonx_gov/metrics/ndcg/ndcg_metric.py +166 -0
  125. ibm_watsonx_gov/metrics/output_token_count/__init__.py +8 -0
  126. ibm_watsonx_gov/metrics/output_token_count/output_token_count_decorator.py +58 -0
  127. ibm_watsonx_gov/metrics/output_token_count/output_token_count_metric.py +112 -0
  128. ibm_watsonx_gov/metrics/pii/__init__.py +16 -0
  129. ibm_watsonx_gov/metrics/pii/input_pii_metric.py +102 -0
  130. ibm_watsonx_gov/metrics/pii/output_pii_metric.py +107 -0
  131. ibm_watsonx_gov/metrics/pii/pii_decorator.py +59 -0
  132. ibm_watsonx_gov/metrics/pii/pii_metric.py +96 -0
  133. ibm_watsonx_gov/metrics/profanity/__init__.py +8 -0
  134. ibm_watsonx_gov/metrics/profanity/profanity_decorator.py +60 -0
  135. ibm_watsonx_gov/metrics/profanity/profanity_metric.py +103 -0
  136. ibm_watsonx_gov/metrics/prompt_safety_risk/__init__.py +8 -0
  137. ibm_watsonx_gov/metrics/prompt_safety_risk/prompt_safety_risk_decorator.py +57 -0
  138. ibm_watsonx_gov/metrics/prompt_safety_risk/prompt_safety_risk_metric.py +128 -0
  139. ibm_watsonx_gov/metrics/reciprocal_rank/__init__.py +0 -0
  140. ibm_watsonx_gov/metrics/reciprocal_rank/reciprocal_rank_decorator.py +62 -0
  141. ibm_watsonx_gov/metrics/reciprocal_rank/reciprocal_rank_metric.py +162 -0
  142. ibm_watsonx_gov/metrics/regex_detection/regex_detection_decorator.py +58 -0
  143. ibm_watsonx_gov/metrics/regex_detection/regex_detection_metric.py +106 -0
  144. ibm_watsonx_gov/metrics/retrieval_precision/__init__.py +0 -0
  145. ibm_watsonx_gov/metrics/retrieval_precision/retrieval_precision_decorator.py +62 -0
  146. ibm_watsonx_gov/metrics/retrieval_precision/retrieval_precision_metric.py +170 -0
  147. ibm_watsonx_gov/metrics/sexual_content/__init__.py +8 -0
  148. ibm_watsonx_gov/metrics/sexual_content/sexual_content_decorator.py +61 -0
  149. ibm_watsonx_gov/metrics/sexual_content/sexual_content_metric.py +103 -0
  150. ibm_watsonx_gov/metrics/social_bias/__init__.py +8 -0
  151. ibm_watsonx_gov/metrics/social_bias/social_bias_decorator.py +62 -0
  152. ibm_watsonx_gov/metrics/social_bias/social_bias_metric.py +103 -0
  153. ibm_watsonx_gov/metrics/status/__init__.py +0 -0
  154. ibm_watsonx_gov/metrics/status/status_metric.py +113 -0
  155. ibm_watsonx_gov/metrics/text_grade_level/__init__.py +8 -0
  156. ibm_watsonx_gov/metrics/text_grade_level/text_grade_level_decorator.py +59 -0
  157. ibm_watsonx_gov/metrics/text_grade_level/text_grade_level_metric.py +127 -0
  158. ibm_watsonx_gov/metrics/text_reading_ease/__init__.py +8 -0
  159. ibm_watsonx_gov/metrics/text_reading_ease/text_reading_ease_decorator.py +59 -0
  160. ibm_watsonx_gov/metrics/text_reading_ease/text_reading_ease_metric.py +123 -0
  161. ibm_watsonx_gov/metrics/tool_call_accuracy/__init__.py +0 -0
  162. ibm_watsonx_gov/metrics/tool_call_accuracy/tool_call_accuracy_decorator.py +67 -0
  163. ibm_watsonx_gov/metrics/tool_call_accuracy/tool_call_accuracy_metric.py +162 -0
  164. ibm_watsonx_gov/metrics/tool_call_parameter_accuracy/__init__.py +0 -0
  165. ibm_watsonx_gov/metrics/tool_call_parameter_accuracy/tool_call_parameter_accuracy_decorator.py +68 -0
  166. ibm_watsonx_gov/metrics/tool_call_parameter_accuracy/tool_call_parameter_accuracy_metric.py +151 -0
  167. ibm_watsonx_gov/metrics/tool_call_relevance/__init__.py +0 -0
  168. ibm_watsonx_gov/metrics/tool_call_relevance/tool_call_relevance_decorator.py +71 -0
  169. ibm_watsonx_gov/metrics/tool_call_relevance/tool_call_relevance_metric.py +166 -0
  170. ibm_watsonx_gov/metrics/tool_call_syntactic_accuracy/__init__.py +0 -0
  171. ibm_watsonx_gov/metrics/tool_call_syntactic_accuracy/tool_call_syntactic_accuracy_decorator.py +66 -0
  172. ibm_watsonx_gov/metrics/tool_call_syntactic_accuracy/tool_call_syntactic_accuracy_metric.py +121 -0
  173. ibm_watsonx_gov/metrics/topic_relevance/__init__.py +8 -0
  174. ibm_watsonx_gov/metrics/topic_relevance/topic_relevance_decorator.py +57 -0
  175. ibm_watsonx_gov/metrics/topic_relevance/topic_relevance_metric.py +106 -0
  176. ibm_watsonx_gov/metrics/unethical_behavior/__init__.py +8 -0
  177. ibm_watsonx_gov/metrics/unethical_behavior/unethical_behavior_decorator.py +61 -0
  178. ibm_watsonx_gov/metrics/unethical_behavior/unethical_behavior_metric.py +103 -0
  179. ibm_watsonx_gov/metrics/unsuccessful_requests/__init__.py +0 -0
  180. ibm_watsonx_gov/metrics/unsuccessful_requests/unsuccessful_requests_decorator.py +66 -0
  181. ibm_watsonx_gov/metrics/unsuccessful_requests/unsuccessful_requests_metric.py +128 -0
  182. ibm_watsonx_gov/metrics/user_id/__init__.py +0 -0
  183. ibm_watsonx_gov/metrics/user_id/user_id_metric.py +111 -0
  184. ibm_watsonx_gov/metrics/utils.py +440 -0
  185. ibm_watsonx_gov/metrics/violence/__init__.py +8 -0
  186. ibm_watsonx_gov/metrics/violence/violence_decorator.py +60 -0
  187. ibm_watsonx_gov/metrics/violence/violence_metric.py +103 -0
  188. ibm_watsonx_gov/prompt_evaluator/__init__.py +9 -0
  189. ibm_watsonx_gov/prompt_evaluator/impl/__init__.py +8 -0
  190. ibm_watsonx_gov/prompt_evaluator/impl/prompt_evaluator_impl.py +554 -0
  191. ibm_watsonx_gov/prompt_evaluator/impl/pta_lifecycle_evaluator.py +2332 -0
  192. ibm_watsonx_gov/prompt_evaluator/prompt_evaluator.py +262 -0
  193. ibm_watsonx_gov/providers/__init__.py +8 -0
  194. ibm_watsonx_gov/providers/detectors_provider.cp313-win_amd64.pyd +0 -0
  195. ibm_watsonx_gov/providers/detectors_provider.py +415 -0
  196. ibm_watsonx_gov/providers/eval_assist_provider.cp313-win_amd64.pyd +0 -0
  197. ibm_watsonx_gov/providers/eval_assist_provider.py +266 -0
  198. ibm_watsonx_gov/providers/inference_engines/__init__.py +0 -0
  199. ibm_watsonx_gov/providers/inference_engines/custom_inference_engine.py +165 -0
  200. ibm_watsonx_gov/providers/inference_engines/portkey_inference_engine.py +57 -0
  201. ibm_watsonx_gov/providers/llmevalkit/__init__.py +0 -0
  202. ibm_watsonx_gov/providers/llmevalkit/ciso_agent/main.py +516 -0
  203. ibm_watsonx_gov/providers/llmevalkit/ciso_agent/preprocess_log.py +111 -0
  204. ibm_watsonx_gov/providers/llmevalkit/ciso_agent/utils.py +186 -0
  205. ibm_watsonx_gov/providers/llmevalkit/function_calling/README.md +411 -0
  206. ibm_watsonx_gov/providers/llmevalkit/function_calling/__init__.py +27 -0
  207. ibm_watsonx_gov/providers/llmevalkit/function_calling/comparison/README.md +306 -0
  208. ibm_watsonx_gov/providers/llmevalkit/function_calling/comparison/__init__.py +89 -0
  209. ibm_watsonx_gov/providers/llmevalkit/function_calling/comparison/comparators/__init__.py +30 -0
  210. ibm_watsonx_gov/providers/llmevalkit/function_calling/comparison/comparators/base.py +411 -0
  211. ibm_watsonx_gov/providers/llmevalkit/function_calling/comparison/comparators/code_agent.py +1254 -0
  212. ibm_watsonx_gov/providers/llmevalkit/function_calling/comparison/comparators/exact_match.py +134 -0
  213. ibm_watsonx_gov/providers/llmevalkit/function_calling/comparison/comparators/fuzzy_string.py +104 -0
  214. ibm_watsonx_gov/providers/llmevalkit/function_calling/comparison/comparators/hybrid.py +516 -0
  215. ibm_watsonx_gov/providers/llmevalkit/function_calling/comparison/comparators/llm_judge.py +1882 -0
  216. ibm_watsonx_gov/providers/llmevalkit/function_calling/comparison/pipeline.py +387 -0
  217. ibm_watsonx_gov/providers/llmevalkit/function_calling/comparison/types.py +178 -0
  218. ibm_watsonx_gov/providers/llmevalkit/function_calling/comparison/utils.py +298 -0
  219. ibm_watsonx_gov/providers/llmevalkit/function_calling/consts.py +33 -0
  220. ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/__init__.py +31 -0
  221. ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/base.py +26 -0
  222. ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/function_call/__init__.py +4 -0
  223. ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/function_call/general.py +46 -0
  224. ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/function_call/general_metrics.json +783 -0
  225. ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/function_call/general_metrics_runtime.json +580 -0
  226. ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/function_selection/__init__.py +6 -0
  227. ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/function_selection/function_selection.py +28 -0
  228. ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/function_selection/function_selection_metrics.json +599 -0
  229. ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/function_selection/function_selection_metrics_runtime.json +477 -0
  230. ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/loader.py +259 -0
  231. ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/parameter/__init__.py +7 -0
  232. ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/parameter/parameter.py +52 -0
  233. ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/parameter/parameter_metrics.json +613 -0
  234. ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/parameter/parameter_metrics_runtime.json +489 -0
  235. ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/trajectory/__init__.py +7 -0
  236. ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/trajectory/trajectory.py +43 -0
  237. ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/trajectory/trajectory_metrics.json +161 -0
  238. ibm_watsonx_gov/providers/llmevalkit/function_calling/pipeline/__init__.py +0 -0
  239. ibm_watsonx_gov/providers/llmevalkit/function_calling/pipeline/adapters.py +102 -0
  240. ibm_watsonx_gov/providers/llmevalkit/function_calling/pipeline/pipeline.py +355 -0
  241. ibm_watsonx_gov/providers/llmevalkit/function_calling/pipeline/semantic_checker.py +816 -0
  242. ibm_watsonx_gov/providers/llmevalkit/function_calling/pipeline/static_checker.py +297 -0
  243. ibm_watsonx_gov/providers/llmevalkit/function_calling/pipeline/transformation_prompts.py +509 -0
  244. ibm_watsonx_gov/providers/llmevalkit/function_calling/pipeline/types.py +596 -0
  245. ibm_watsonx_gov/providers/llmevalkit/llm/README.md +375 -0
  246. ibm_watsonx_gov/providers/llmevalkit/llm/__init__.py +137 -0
  247. ibm_watsonx_gov/providers/llmevalkit/llm/base.py +426 -0
  248. ibm_watsonx_gov/providers/llmevalkit/llm/output_parser.py +364 -0
  249. ibm_watsonx_gov/providers/llmevalkit/llm/providers/__init__.py +0 -0
  250. ibm_watsonx_gov/providers/llmevalkit/llm/providers/consts.py +7 -0
  251. ibm_watsonx_gov/providers/llmevalkit/llm/providers/ibm_watsonx_ai/__init__.py +0 -0
  252. ibm_watsonx_gov/providers/llmevalkit/llm/providers/ibm_watsonx_ai/ibm_watsonx_ai.py +656 -0
  253. ibm_watsonx_gov/providers/llmevalkit/llm/providers/litellm/__init__.py +0 -0
  254. ibm_watsonx_gov/providers/llmevalkit/llm/providers/litellm/litellm.py +509 -0
  255. ibm_watsonx_gov/providers/llmevalkit/llm/providers/litellm/rits.py +224 -0
  256. ibm_watsonx_gov/providers/llmevalkit/llm/providers/litellm/watsonx.py +60 -0
  257. ibm_watsonx_gov/providers/llmevalkit/llm/providers/mock_llm_client.py +75 -0
  258. ibm_watsonx_gov/providers/llmevalkit/llm/providers/openai/__init__.py +0 -0
  259. ibm_watsonx_gov/providers/llmevalkit/llm/providers/openai/openai.py +639 -0
  260. ibm_watsonx_gov/providers/llmevalkit/llm/providers/wxo_ai_gateway/__init__.py +0 -0
  261. ibm_watsonx_gov/providers/llmevalkit/llm/providers/wxo_ai_gateway/wxo_ai_gateway.py +134 -0
  262. ibm_watsonx_gov/providers/llmevalkit/llm/providers/wxo_ai_gateway/wxo_ai_gateway_inference.py +214 -0
  263. ibm_watsonx_gov/providers/llmevalkit/llm/types.py +136 -0
  264. ibm_watsonx_gov/providers/llmevalkit/metrics/__init__.py +4 -0
  265. ibm_watsonx_gov/providers/llmevalkit/metrics/field.py +255 -0
  266. ibm_watsonx_gov/providers/llmevalkit/metrics/metric.py +332 -0
  267. ibm_watsonx_gov/providers/llmevalkit/metrics/metrics_runner.py +188 -0
  268. ibm_watsonx_gov/providers/llmevalkit/metrics/prompt.py +403 -0
  269. ibm_watsonx_gov/providers/llmevalkit/metrics/utils.py +46 -0
  270. ibm_watsonx_gov/providers/llmevalkit/prompt/__init__.py +0 -0
  271. ibm_watsonx_gov/providers/llmevalkit/prompt/runner.py +144 -0
  272. ibm_watsonx_gov/providers/tool_call_metric_provider.py +455 -0
  273. ibm_watsonx_gov/providers/unitxt_provider.cp313-win_amd64.pyd +0 -0
  274. ibm_watsonx_gov/tools/__init__.py +10 -0
  275. ibm_watsonx_gov/tools/clients/__init__.py +11 -0
  276. ibm_watsonx_gov/tools/clients/ai_tool_client.py +405 -0
  277. ibm_watsonx_gov/tools/clients/detector_client.py +82 -0
  278. ibm_watsonx_gov/tools/core/__init__.py +8 -0
  279. ibm_watsonx_gov/tools/core/tool_loader.py +237 -0
  280. ibm_watsonx_gov/tools/entities/__init__.py +8 -0
  281. ibm_watsonx_gov/tools/entities/ai_tools.py +435 -0
  282. ibm_watsonx_gov/tools/onboarding/create/answer_relevance_detector.json +57 -0
  283. ibm_watsonx_gov/tools/onboarding/create/chromadb_retrieval_tool.json +63 -0
  284. ibm_watsonx_gov/tools/onboarding/create/context_relevance_detector.json +57 -0
  285. ibm_watsonx_gov/tools/onboarding/create/duduckgo_search_tool.json +53 -0
  286. ibm_watsonx_gov/tools/onboarding/create/google_search_tool.json +62 -0
  287. ibm_watsonx_gov/tools/onboarding/create/hap_detector.json +70 -0
  288. ibm_watsonx_gov/tools/onboarding/create/jailbreak_detector.json +70 -0
  289. ibm_watsonx_gov/tools/onboarding/create/pii_detector.json +36 -0
  290. ibm_watsonx_gov/tools/onboarding/create/prompt_safety_risk_detector.json +69 -0
  291. ibm_watsonx_gov/tools/onboarding/create/topic_relevance_detector.json +57 -0
  292. ibm_watsonx_gov/tools/onboarding/create/weather_tool.json +39 -0
  293. ibm_watsonx_gov/tools/onboarding/create/webcrawler_tool.json +34 -0
  294. ibm_watsonx_gov/tools/onboarding/create/wikipedia_search_tool.json +53 -0
  295. ibm_watsonx_gov/tools/onboarding/delete/delete_tools.json +4 -0
  296. ibm_watsonx_gov/tools/onboarding/update/google_search_tool.json +38 -0
  297. ibm_watsonx_gov/tools/ootb/__init__.py +8 -0
  298. ibm_watsonx_gov/tools/ootb/detectors/__init__.py +8 -0
  299. ibm_watsonx_gov/tools/ootb/detectors/hap_detector_tool.py +109 -0
  300. ibm_watsonx_gov/tools/ootb/detectors/jailbreak_detector_tool.py +104 -0
  301. ibm_watsonx_gov/tools/ootb/detectors/pii_detector_tool.py +83 -0
  302. ibm_watsonx_gov/tools/ootb/detectors/prompt_safety_risk_detector_tool.py +111 -0
  303. ibm_watsonx_gov/tools/ootb/detectors/topic_relevance_detector_tool.py +101 -0
  304. ibm_watsonx_gov/tools/ootb/rag/__init__.py +8 -0
  305. ibm_watsonx_gov/tools/ootb/rag/answer_relevance_detector_tool.py +119 -0
  306. ibm_watsonx_gov/tools/ootb/rag/context_relevance_detector_tool.py +118 -0
  307. ibm_watsonx_gov/tools/ootb/search/__init__.py +8 -0
  308. ibm_watsonx_gov/tools/ootb/search/duckduckgo_search_tool.py +62 -0
  309. ibm_watsonx_gov/tools/ootb/search/google_search_tool.py +105 -0
  310. ibm_watsonx_gov/tools/ootb/search/weather_tool.py +95 -0
  311. ibm_watsonx_gov/tools/ootb/search/web_crawler_tool.py +69 -0
  312. ibm_watsonx_gov/tools/ootb/search/wikipedia_search_tool.py +63 -0
  313. ibm_watsonx_gov/tools/ootb/vectordb/__init__.py +8 -0
  314. ibm_watsonx_gov/tools/ootb/vectordb/chromadb_retriever_tool.py +111 -0
  315. ibm_watsonx_gov/tools/rest_api/__init__.py +10 -0
  316. ibm_watsonx_gov/tools/rest_api/restapi_tool.py +72 -0
  317. ibm_watsonx_gov/tools/schemas/__init__.py +10 -0
  318. ibm_watsonx_gov/tools/schemas/search_tool_schema.py +46 -0
  319. ibm_watsonx_gov/tools/schemas/vectordb_retrieval_schema.py +55 -0
  320. ibm_watsonx_gov/tools/utils/__init__.py +14 -0
  321. ibm_watsonx_gov/tools/utils/constants.py +69 -0
  322. ibm_watsonx_gov/tools/utils/display_utils.py +38 -0
  323. ibm_watsonx_gov/tools/utils/environment.py +108 -0
  324. ibm_watsonx_gov/tools/utils/package_utils.py +40 -0
  325. ibm_watsonx_gov/tools/utils/platform_url_mapping.cp313-win_amd64.pyd +0 -0
  326. ibm_watsonx_gov/tools/utils/python_utils.py +68 -0
  327. ibm_watsonx_gov/tools/utils/tool_utils.py +206 -0
  328. ibm_watsonx_gov/traces/__init__.py +8 -0
  329. ibm_watsonx_gov/traces/span_exporter.py +195 -0
  330. ibm_watsonx_gov/traces/span_node.py +251 -0
  331. ibm_watsonx_gov/traces/span_util.py +153 -0
  332. ibm_watsonx_gov/traces/trace_utils.py +1074 -0
  333. ibm_watsonx_gov/utils/__init__.py +8 -0
  334. ibm_watsonx_gov/utils/aggregation_util.py +346 -0
  335. ibm_watsonx_gov/utils/async_util.py +62 -0
  336. ibm_watsonx_gov/utils/authenticator.py +144 -0
  337. ibm_watsonx_gov/utils/constants.py +15 -0
  338. ibm_watsonx_gov/utils/errors.py +40 -0
  339. ibm_watsonx_gov/utils/gov_sdk_logger.py +39 -0
  340. ibm_watsonx_gov/utils/insights_generator.py +1285 -0
  341. ibm_watsonx_gov/utils/python_utils.py +425 -0
  342. ibm_watsonx_gov/utils/rest_util.py +73 -0
  343. ibm_watsonx_gov/utils/segment_batch_manager.py +162 -0
  344. ibm_watsonx_gov/utils/singleton_meta.py +25 -0
  345. ibm_watsonx_gov/utils/url_mapping.cp313-win_amd64.pyd +0 -0
  346. ibm_watsonx_gov/utils/validation_util.py +126 -0
  347. ibm_watsonx_gov/visualizations/__init__.py +13 -0
  348. ibm_watsonx_gov/visualizations/metric_descriptions.py +57 -0
  349. ibm_watsonx_gov/visualizations/model_insights.py +1304 -0
  350. ibm_watsonx_gov/visualizations/visualization_utils.py +75 -0
  351. ibm_watsonx_gov-1.3.3.dist-info/METADATA +93 -0
  352. ibm_watsonx_gov-1.3.3.dist-info/RECORD +353 -0
  353. ibm_watsonx_gov-1.3.3.dist-info/WHEEL +4 -0
@@ -0,0 +1,53 @@
1
+ {
2
+ "display_name": "Wikipedia Search",
3
+ "tool_name": "wikipedia_search_tool",
4
+ "service_provider_type": "IBM",
5
+ "description": "Search Wikipedia and return results for given query",
6
+ "inventory_id": "<attribute_value>",
7
+ "reusable": "true",
8
+ "category": ["Search"],
9
+ "framework": ["langchain","langgraph"],
10
+ "code": {
11
+ "source_code_base64": "",
12
+ "run_time_details": {
13
+ "engine": "Python >=3.11"
14
+ }
15
+ },
16
+ "config":{
17
+ "description": "Configuration schema for OOTB search tools",
18
+ "properties": {
19
+ "top_k_results": {
20
+ "default": 3,
21
+ "description": "Number of search results to retrieve",
22
+ "title": "Top K Results",
23
+ "type": "integer"
24
+ }
25
+ },
26
+ "title": "SearchToolConfig",
27
+ "type": "object"
28
+ },
29
+ "schema":{
30
+ "properties": {
31
+ "query": {
32
+ "description": "Search the input from the given search tool",
33
+ "title": "Query",
34
+ "type": "string"
35
+ },
36
+ "top_k_results": {
37
+ "default": 3,
38
+ "description": "Number of search results to retrieve",
39
+ "title": "Top K Results",
40
+ "type": "integer"
41
+ }
42
+ },
43
+ "required": [
44
+ "query"
45
+ ],
46
+ "title": "SearchToolInput",
47
+ "type": "object"
48
+ },
49
+ "dependencies": {
50
+ "remote_services":[],
51
+ "run_time_packages":["wikipedia"]
52
+ }
53
+ }
@@ -0,0 +1,4 @@
1
+ {
2
+ "tool_names": [
3
+ ]
4
+ }
@@ -0,0 +1,38 @@
1
+ {
2
+ "tool_name": "google_search_tool",
3
+ "version": "1.1.0",
4
+ "patch_tool": [
5
+ {
6
+ "op": "replace",
7
+ "path": "/config",
8
+ "value": {
9
+ "description": "Configuration schema for OOTB search tools",
10
+ "properties": {
11
+ "top_k_results": {
12
+ "default": 3,
13
+ "description": "Number of search results to retrieve",
14
+ "title": "Top K Results",
15
+ "type": "integer"
16
+ },
17
+ "serpapi_key": {
18
+ "description": "API key used to authenticate requests to SerpAPI. You can generate your API key from https://serpapi.com/manage-api-key.",
19
+ "title": "SerpAPI Key",
20
+ "type": "string"
21
+ }
22
+ },
23
+ "title": "SearchToolConfig",
24
+ "type": "object",
25
+ "required": [
26
+ "serpapi_key"
27
+ ]
28
+ }
29
+ },
30
+ {
31
+ "op": "replace",
32
+ "path": "/dependencies/run_time_packages",
33
+ "value": [
34
+ "google-search-results"
35
+ ]
36
+ }
37
+ ]
38
+ }
@@ -0,0 +1,8 @@
1
+ # ----------------------------------------------------------------------------------------------------
2
+ # IBM Confidential
3
+ # Licensed Materials - Property of IBM
4
+ # 5737-H76, 5900-A3Q
5
+ # © Copyright IBM Corp. 2025 All Rights Reserved.
6
+ # US Government Users Restricted Rights - Use, duplication or disclosure restricted by
7
+ # GSA ADPSchedule Contract with IBM Corp.
8
+ # ----------------------------------------------------------------------------------------------------
@@ -0,0 +1,8 @@
1
+ # ----------------------------------------------------------------------------------------------------
2
+ # IBM Confidential
3
+ # Licensed Materials - Property of IBM
4
+ # 5737-H76, 5900-A3Q
5
+ # © Copyright IBM Corp. 2025 All Rights Reserved.
6
+ # US Government Users Restricted Rights - Use, duplication or disclosure restricted by
7
+ # GSA ADPSchedule Contract with IBM Corp.
8
+ # ----------------------------------------------------------------------------------------------------
@@ -0,0 +1,109 @@
1
+ # ----------------------------------------------------------------------------------------------------
2
+ # IBM Confidential
3
+ # Licensed Materials - Property of IBM
4
+ # 5737-H76, 5900-A3Q
5
+ # © Copyright IBM Corp. 2025 All Rights Reserved.
6
+ # US Government Users Restricted Rights - Use, duplication or disclosure restricted by
7
+ # GSA ADPSchedule Contract with IBM Corp.
8
+ # ----------------------------------------------------------------------------------------------------
9
+
10
+ import ibm_watsonx_gov.tools.clients.detector_client as dc
11
+
12
+ from typing import Annotated, List, Optional, Type
13
+ from langchain.callbacks.manager import CallbackManagerForToolRun
14
+ from langchain.tools import BaseTool
15
+ from pydantic import BaseModel, Field, PrivateAttr
16
+
17
+
18
+
19
+
20
+ class HAPInput(BaseModel):
21
+ """
22
+ Model that can be used for setting input args for HAPDetectorTool
23
+ """
24
+ input: Annotated[str,
25
+ Field(..., description="Detect HAP for the given input")]
26
+ threshold: Annotated[Optional[float], Field(
27
+ 0.8, description="Threshold for hap detection in the input, values range from 0.0 to 1.0")]
28
+
29
+
30
+ class HAPConfig(BaseModel):
31
+ """
32
+ Model that can be used for setting threshold for HAPDetectorTool
33
+ """
34
+ threshold: Annotated[Optional[float], Field(
35
+ 0.8, description="Threshold for hap detection in the input, values range from 0.0 to 1.0")]
36
+
37
+
38
+ class HAPDetectorTool(BaseTool):
39
+ """
40
+ Tool to detect Hate, Abuse and Profanity content in input string
41
+
42
+ Examples:
43
+ Basic usage
44
+ .. code-block:: python
45
+
46
+ hap_detector_tool = HAPDetectorTool()
47
+ hap_detector_tool.invoke({"input":"<USER_INPUT>"})
48
+ """
49
+ name: str = "hap_detector"
50
+ description: str = "Tool to detect Hate, Abuse and Profanity content in the input string"
51
+ args_schema: Type[BaseModel] = HAPInput
52
+
53
+ _url: any = PrivateAttr()
54
+ _threshold: any = PrivateAttr()
55
+
56
+ def __init__(self, **kwargs):
57
+ super().__init__(**kwargs)
58
+ config = HAPConfig(**kwargs)
59
+ self._threshold = config.threshold
60
+ self._url = dc.DETECTIONS_URL.format(dc.get_base_url())
61
+ self._base_payload = dc.get_base_payload(dc.HAP,detector_params= {
62
+ "threshold": 0.0
63
+ })
64
+
65
+ # Define HAP Tool
66
+ def _run(self,
67
+ input: str,
68
+ run_manager: Optional[CallbackManagerForToolRun] = None,
69
+ **kwargs) -> List[str]:
70
+ """
71
+ Detects Hate, Abuse and Profanity content in the input string
72
+
73
+ Sample Response:
74
+ {
75
+ 'detections': [{'start': 0,
76
+ 'end': 46,
77
+ 'text': '<USER_INPUT>',
78
+ 'detection_type': 'hap',
79
+ 'detection': 'has_HAP',
80
+ 'score': 0.9495730400085448,
81
+ }],
82
+ 'is_hap_detected': True
83
+ }
84
+ """
85
+
86
+ threshold = kwargs.get("threshold",self._threshold)
87
+ # By default threshold is 0.0 and score is compared with the default/user provided threshold
88
+
89
+ payload = dc.get_payload(self._base_payload,detector_payload={
90
+ "input":input
91
+ })
92
+ response = dc.call_detections(self._url, payload)
93
+
94
+
95
+ #Add boolean to response
96
+ response["is_hap_detected"] = False
97
+ for detection in response.get("detections"):
98
+ if detection.get("score") > threshold:
99
+ response["is_hap_detected"] = True
100
+ break
101
+
102
+ return response
103
+
104
+ async def _arun(self,
105
+ input: str,
106
+ run_manager: Optional[CallbackManagerForToolRun] = None,
107
+ **kwargs) -> List[str]:
108
+
109
+ return self._run(input , kwargs)
@@ -0,0 +1,104 @@
1
+ # ----------------------------------------------------------------------------------------------------
2
+ # IBM Confidential
3
+ # Licensed Materials - Property of IBM
4
+ # 5737-H76, 5900-A3Q
5
+ # © Copyright IBM Corp. 2025 All Rights Reserved.
6
+ # US Government Users Restricted Rights - Use, duplication or disclosure restricted by
7
+ # GSA ADPSchedule Contract with IBM Corp.
8
+ # ----------------------------------------------------------------------------------------------------
9
+
10
+ import json
11
+
12
+ import ibm_watsonx_gov.tools.clients.detector_client as dc
13
+
14
+ from typing import Annotated, List, Optional, Type
15
+
16
+ from langchain.callbacks.manager import CallbackManagerForToolRun
17
+ from langchain.tools import BaseTool
18
+ from pydantic import BaseModel, Field, PrivateAttr
19
+
20
+
21
+ class JailBreakInput(BaseModel):
22
+ """
23
+ Model that can be used for setting input args for JailBreakDetectorTool
24
+ """
25
+ input: Annotated[str,
26
+ Field(..., description="Detect JailBreak for the given input")]
27
+ threshold: Annotated[Optional[float], Field(
28
+ 0.8, description="Threshold for detecting jailbreak in the input, values range from 0.0 to 1.0")]
29
+
30
+
31
+ class JailBreakConfig(BaseModel):
32
+ """
33
+ Model that can be used for setting threshold for JailBreakDetectorTool
34
+ """
35
+ threshold: Annotated[Optional[float], Field(
36
+ 0.8, description="Threshold for detecting jailbreak in the input, values range from 0.0 to 1.0")]
37
+
38
+
39
+ class JailBreakDetectorTool(BaseTool):
40
+ """
41
+ Tool to detect deliberate circumvention of AI systems built-in safeguards or ethical guidelines.
42
+ This involves crafting specific prompts or scenarios designed to manipulate the AI into generating
43
+ restricted or inappropriate content.
44
+
45
+ Examples:
46
+ Basic usage
47
+ .. code-block:: python
48
+
49
+ jailbreak_detector_tool = JailBreakDetectorTool()
50
+ jailbreak_detector_tool.invoke({"input":"<USER_INPUT>"})
51
+ """
52
+
53
+ name: str = "jailbreak_detector"
54
+ description: str = """Tool that detects deliberate circumvention of AI systems built-in safeguards or ethical guidelines.
55
+ This involves crafting specific prompts or scenarios designed to manipulate the AI into generating
56
+ restricted or inappropriate content."""
57
+ args_schema: Type[BaseModel] = JailBreakInput
58
+
59
+ _url: any = PrivateAttr()
60
+ _threshold: any = PrivateAttr()
61
+
62
+ def __init__(self, **kwargs):
63
+ super().__init__(**kwargs)
64
+ config = JailBreakConfig(**kwargs)
65
+ self._threshold = config.threshold
66
+ self._url = dc.DETECTIONS_URL.format(dc.get_base_url())
67
+ self._base_payload = dc.get_base_payload(dc.GRANITE_GUADIAN,detector_params= {
68
+ "risk_name": "jailbreak",
69
+ "threshold": 0.0
70
+ })
71
+
72
+ # Define JailBreak Tool
73
+ def _run(self,
74
+ input: str,
75
+ run_manager: Optional[CallbackManagerForToolRun] = None,
76
+ **kwargs) -> List[str]:
77
+ """
78
+ Sample Response:
79
+ {'detections': [{'start': 0,
80
+ 'end': 26,
81
+ 'text': '<USER_INPUT>',
82
+ 'detection_type': 'risk',
83
+ 'detection': 'Yes',
84
+ 'score': 0.9688562154769896}],
85
+ 'is_jailbreak_detected': True}
86
+ """
87
+ threshold = kwargs.get("threshold",self._threshold)
88
+ payload = dc.get_payload(self._base_payload,detector_payload={
89
+ "input":input
90
+ })
91
+ response = dc.call_detections(self._url, payload)
92
+
93
+ if response.get("detections")[0].get("score") > threshold:
94
+ response["is_jailbreak_detected"] = True
95
+ else:
96
+ response["is_jailbreak_detected"] = False
97
+ return response
98
+
99
+ async def _arun(self,
100
+ input: str,
101
+ run_manager: Optional[CallbackManagerForToolRun] = None,
102
+ **kwargs) -> List[str]:
103
+
104
+ return self._run(input , kwargs)
@@ -0,0 +1,83 @@
1
+ # ----------------------------------------------------------------------------------------------------
2
+ # IBM Confidential
3
+ # Licensed Materials - Property of IBM
4
+ # 5737-H76, 5900-A3Q
5
+ # © Copyright IBM Corp. 2025 All Rights Reserved.
6
+ # US Government Users Restricted Rights - Use, duplication or disclosure restricted by
7
+ # GSA ADPSchedule Contract with IBM Corp.
8
+ # ----------------------------------------------------------------------------------------------------
9
+
10
+ import ibm_watsonx_gov.tools.clients.detector_client as dc
11
+
12
+ from typing import Annotated, List, Optional, Type
13
+ from langchain.callbacks.manager import CallbackManagerForToolRun
14
+ from langchain.tools import BaseTool
15
+ from pydantic import BaseModel, Field, PrivateAttr
16
+
17
+
18
+ class PIIInput(BaseModel):
19
+ """
20
+ Model that can be used for setting input args for PIIDetectorTool
21
+ """
22
+ input: Annotated[str,
23
+ Field(..., description="Detect PII for the given input")]
24
+
25
+
26
+ class PIIDetectorTool(BaseTool):
27
+ """
28
+ Tool to detect PII content in input string
29
+
30
+ Examples:
31
+ Basic usage
32
+ .. code-block:: python
33
+
34
+ pii_detector_tool = PIIDetectorTool()
35
+ pii_detector_tool.invoke({"input":"<USER_INPUT>"})
36
+ """
37
+ name: str = "pii_detector"
38
+ description: str = "Tool that detects Personally Identifiable Information(PII) information in the input string."
39
+ args_schema: Type[BaseModel] = PIIInput
40
+ _url: any = PrivateAttr()
41
+
42
+ def __init__(self, **kwargs):
43
+ super().__init__(**kwargs)
44
+ self._url = dc.DETECTIONS_URL.format(dc.get_base_url())
45
+ self._base_payload = dc.get_base_payload(dc.PII)
46
+
47
+ # Define PII Tool
48
+ def _run(self,
49
+ input: str,
50
+ run_manager: Optional[CallbackManagerForToolRun] = None,
51
+ **kwargs) -> List[str]:
52
+
53
+ """
54
+ Sample Response:
55
+ {'detections': [{'start': 15,
56
+ 'end': 30,
57
+ 'text': 'johndoe@abc.com',
58
+ 'detection_type': 'pii',
59
+ 'detection': 'EmailAddress',
60
+ 'score': 0.8}],
61
+ 'is_pii_detected': True}
62
+ """
63
+
64
+ payload = dc.get_payload(self._base_payload,detector_payload={
65
+ "input":input
66
+ })
67
+ response = dc.call_detections(self._url, payload)
68
+
69
+
70
+ if response.get("detections") == []:
71
+ # No PII information detected
72
+ response["is_pii_detected"] = False
73
+ else:
74
+ response["is_pii_detected"] = True
75
+
76
+ return response
77
+
78
+ async def _arun(self,
79
+ input: str,
80
+ run_manager: Optional[CallbackManagerForToolRun] = None,
81
+ **kwargs) -> List[str]:
82
+
83
+ return self._run(input , kwargs)
@@ -0,0 +1,111 @@
1
+ # ----------------------------------------------------------------------------------------------------
2
+ # IBM Confidential
3
+ # Licensed Materials - Property of IBM
4
+ # 5737-H76, 5900-A3Q
5
+ # © Copyright IBM Corp. 2025 All Rights Reserved.
6
+ # US Government Users Restricted Rights - Use, duplication or disclosure restricted by
7
+ # GSA ADPSchedule Contract with IBM Corp.
8
+ # ----------------------------------------------------------------------------------------------------
9
+
10
+ import json
11
+ import ibm_watsonx_gov.tools.clients.detector_client as dc
12
+ from typing import Annotated, List, Optional, Type
13
+ from langchain.callbacks.manager import CallbackManagerForToolRun
14
+ from langchain.tools import BaseTool
15
+ from pydantic import BaseModel, Field, PrivateAttr
16
+
17
+
18
+ class PromptSafetyRiskInput(BaseModel):
19
+ """
20
+ Model that can be used for setting input args for PromptSafetyRiskTool
21
+ """
22
+ input: Annotated[str,
23
+ Field(..., description="Detect PromptSafetyRisk for the given input")]
24
+ threshold: Annotated[Optional[float], Field(
25
+ default=0.8, description="Threshold for prompt safety risk detection in the input, values range from 0.0 to 1.0")]
26
+ system_prompt: Annotated[str,
27
+ Field(..., description="System prompt to configure the chatbot's behavior.")]
28
+ enable_two_level_detection: Annotated[Optional[bool], Field(
29
+ default=True, description="Enable or disable two level detection")]
30
+
31
+
32
+ class PromptSafetyRiskConfig(BaseModel):
33
+ """
34
+ Model that can be used for setting threshold for PromptSafetyRiskTool
35
+ """
36
+ threshold: Annotated[Optional[float], Field(
37
+ default=0.8, description="Threshold for prompt safety risk detection in the input, values range from 0.0 to 1.0")]
38
+ enable_two_level_detection: Annotated[Optional[bool], Field(
39
+ default=False, description="Enable or disable two level detection")]
40
+
41
+
42
+ class PromptSafetyRiskDetectorTool(BaseTool):
43
+ """
44
+ Prompt Safety Risk tool measures the intent of jailbreak and/or prompt injection in the input sent to the LLM.
45
+
46
+ Examples:
47
+ Basic usage
48
+ .. code-block:: python
49
+
50
+ prompt_safety_detector_tool = PromptSafetyRiskTool()
51
+ prompt_safety_detector_tool.invoke({"input":"<USER_INPUT>", "system_prompt":"<SYSTEM_PROMPT>"})
52
+ """
53
+
54
+ name: str = "prompt_safety_risk_detector"
55
+ description: str = "Tool that detects off topic and prompt injection attempts in user input"
56
+ args_schema: Type[BaseModel] = PromptSafetyRiskInput
57
+
58
+ _threshold: any = PrivateAttr()
59
+ _system_prompt: any = PrivateAttr()
60
+ _url: any = PrivateAttr()
61
+ _enable_two_level_detection: any = PrivateAttr()
62
+
63
+ def __init__(self, **kwargs):
64
+ super().__init__(**kwargs)
65
+ config = PromptSafetyRiskConfig(**kwargs)
66
+ self._threshold = config.threshold
67
+ self._enable_two_level_detection = config.enable_two_level_detection
68
+ self._url = dc.DETECTIONS_URL.format(dc.get_base_url())
69
+
70
+ def _run(self,
71
+ input: str,
72
+ system_prompt:str,
73
+ run_manager: Optional[CallbackManagerForToolRun] = None,
74
+ **kwargs) -> List[str]:
75
+
76
+ """
77
+ Sample Response:
78
+ {'detections': [{'start': 0,
79
+ 'end': 36,
80
+ 'text': '<USER_INPUT>',
81
+ 'detection_type': 'topic_relevance',
82
+ 'detection': 'off-topic',
83
+ 'score': 0.9422797560691832}],
84
+ 'is_prompt_safety_risk_detected': True}
85
+ """
86
+
87
+ threshold = threshold = kwargs.get("threshold",self._threshold)
88
+ enable_two_level_detection = kwargs.get("enable_two_level_detection",self._enable_two_level_detection)
89
+
90
+ # Default or user provided threshold and enable_two_level_detection is passed to the endpoint
91
+ detector_params = {
92
+ "system_prompt": system_prompt,
93
+ "enable_two_level_detection": enable_two_level_detection,
94
+ "threshold": 0.0
95
+ }
96
+
97
+ base_payload = dc.get_base_payload(dc.PROMPT_SAFETY_RISK,detector_params)
98
+
99
+ # payload = get_payload(
100
+ # self.detector_name, input, detector_params=detector_params)
101
+ payload = dc.get_payload(base_payload,detector_payload={
102
+ "input":input
103
+ })
104
+ response = dc.call_detections(self._url, payload)
105
+
106
+
107
+ if len(response.get("detections")) > 0 and response.get("detections")[0].get("score") > threshold:
108
+ response["is_prompt_safety_risk_detected"] = True
109
+ else:
110
+ response["is_prompt_safety_risk_detected"] = False
111
+ return response
@@ -0,0 +1,101 @@
1
+ # ----------------------------------------------------------------------------------------------------
2
+ # IBM Confidential
3
+ # Licensed Materials - Property of IBM
4
+ # 5737-H76, 5900-A3Q
5
+ # © Copyright IBM Corp. 2025 All Rights Reserved.
6
+ # US Government Users Restricted Rights - Use, duplication or disclosure restricted by
7
+ # GSA ADPSchedule Contract with IBM Corp.
8
+ # ----------------------------------------------------------------------------------------------------
9
+
10
+ import json
11
+ import ibm_watsonx_gov.tools.clients.detector_client as dc
12
+
13
+ from typing import Annotated, List, Optional, Type
14
+
15
+ from langchain.callbacks.manager import CallbackManagerForToolRun
16
+ from langchain.tools import BaseTool
17
+ from pydantic import BaseModel, Field, PrivateAttr
18
+
19
+
20
+ class TopicRelevanceDetectorInput(BaseModel):
21
+ """
22
+ Model that can be used for setting input args for TopicRelevanceDetectorTool
23
+ """
24
+ input: Annotated[str,
25
+ Field(..., description="Detect topic relevance for the given input")]
26
+ threshold: Annotated[Optional[float], Field(
27
+ default=0.8, description="Threshold for off-topic detection in the input, values range from 0.0 to 1.0")]
28
+ system_prompt: Annotated[str,
29
+ Field(..., description="System prompt to configure the chatbot's behavior.")]
30
+
31
+
32
+ class TopicRelevanceDetectorConfig(BaseModel):
33
+ """
34
+ Model that can be used for setting threshold for TopicRelevanceDetectorTool
35
+ """
36
+ threshold: Annotated[Optional[float], Field(
37
+ default=0.8, description="Threshold for off-topic detection in the input, values range from 0.0 to 1.0")]
38
+
39
+
40
+ class TopicRelevanceDetectorTool(BaseTool):
41
+ """
42
+ Tool that analyzes whether the user input is relevant to the topic defined in the system prompt and measures the degree of alignment.
43
+
44
+ Examples:
45
+ Basic usage
46
+ .. code-block:: python
47
+
48
+ topic_relevance_detector = TopicRelevanceDetectorTool()
49
+ topic_relevance_detector.invoke({"input":"<USER_INPUT>", "system_prompt":"<SYSTEM_PROMPT>"})
50
+ """
51
+
52
+ name: str = "topic_relevance_detector"
53
+ description: str = "Tool that analyzes whether the user input is relevant to the topic defined in the system prompt and measures the degree of alignment."
54
+ args_schema: Type[BaseModel] = TopicRelevanceDetectorInput
55
+
56
+ _threshold: any = PrivateAttr()
57
+ _system_prompt: any = PrivateAttr()
58
+ _url: any = PrivateAttr()
59
+
60
+ def __init__(self, **kwargs):
61
+ super().__init__(**kwargs)
62
+ config = TopicRelevanceDetectorConfig(**kwargs)
63
+ self._threshold = config.threshold
64
+ self._url = dc.DETECTIONS_URL.format(dc.get_base_url())
65
+
66
+ def _run(self,
67
+ input: str,
68
+ system_prompt,
69
+ run_manager: Optional[CallbackManagerForToolRun] = None,
70
+ **kwargs) -> List[str]:
71
+ """
72
+ Sample Response:
73
+ {'detections': [{'start': 0,
74
+ 'end': 35,
75
+ 'text': '<USER_INPUT>',
76
+ 'detection_type': 'topic_relevance',
77
+ 'detection': 'off-topic',
78
+ 'score': 0.847662091255188}],
79
+ 'is_topic_relevant': False}
80
+ """
81
+
82
+ threshold = kwargs.get("threshold",self._threshold)
83
+ # By default threshold is 0.0 and score is compared with the default/user provided threshold
84
+ detector_params = {
85
+ "system_prompt": system_prompt,
86
+ "threshold": 0.0
87
+ }
88
+
89
+ base_payload = dc.get_base_payload(dc.TOPIC_RELEVANCE,detector_params)
90
+ payload = dc.get_payload(base_payload,detector_payload={
91
+ "input":input
92
+ })
93
+ response = dc.call_detections(self._url, payload)
94
+
95
+
96
+ #Propogate boolean values
97
+ if len(response.get("detections")) > 0 and response.get("detections")[0].get("score") > threshold:
98
+ response["is_topic_relevant"] = False
99
+ else:
100
+ response["is_topic_relevant"] = True
101
+ return response
@@ -0,0 +1,8 @@
1
+ # ----------------------------------------------------------------------------------------------------
2
+ # IBM Confidential
3
+ # Licensed Materials - Property of IBM
4
+ # 5737-H76, 5900-A3Q
5
+ # © Copyright IBM Corp. 2025 All Rights Reserved.
6
+ # US Government Users Restricted Rights - Use, duplication or disclosure restricted by
7
+ # GSA ADPSchedule Contract with IBM Corp.
8
+ # ----------------------------------------------------------------------------------------------------