futureagi 0.4.2__tar.gz → 0.5.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of futureagi might be problematic. Click here for more details.

Files changed (223) hide show
  1. {futureagi-0.4.2 → futureagi-0.5.0}/PKG-INFO +24 -2
  2. {futureagi-0.4.2 → futureagi-0.5.0}/fi/datasets/client.py +73 -76
  3. {futureagi-0.4.2 → futureagi-0.5.0}/fi/evals/__init__.py +2 -0
  4. futureagi-0.5.0/fi/evals/protect.py +409 -0
  5. {futureagi-0.4.2 → futureagi-0.5.0}/fi/evals/templates.py +16 -0
  6. futureagi-0.5.0/fi/integrations/otel/__init__.py +50 -0
  7. futureagi-0.5.0/fi/integrations/otel/frameworks/anthropic/__init__.py +117 -0
  8. futureagi-0.5.0/fi/integrations/otel/frameworks/anthropic/_stream.py +556 -0
  9. futureagi-0.5.0/fi/integrations/otel/frameworks/anthropic/_utils.py +108 -0
  10. futureagi-0.5.0/fi/integrations/otel/frameworks/anthropic/_with_span.py +119 -0
  11. futureagi-0.5.0/fi/integrations/otel/frameworks/anthropic/_wrappers.py +856 -0
  12. futureagi-0.5.0/fi/integrations/otel/frameworks/anthropic/examples/anthropic.sh +21 -0
  13. futureagi-0.5.0/fi/integrations/otel/frameworks/anthropic/examples/end_to_end_tool_use.py +125 -0
  14. futureagi-0.5.0/fi/integrations/otel/frameworks/anthropic/examples/multiple_tool_calling.py +65 -0
  15. futureagi-0.5.0/fi/integrations/otel/frameworks/anthropic/examples/new-1.py +238 -0
  16. futureagi-0.5.0/fi/integrations/otel/frameworks/anthropic/examples/requirements.txt +1 -0
  17. futureagi-0.5.0/fi/integrations/otel/frameworks/anthropic/examples/sync_completions.py +32 -0
  18. futureagi-0.5.0/fi/integrations/otel/frameworks/anthropic/examples/sync_messages.py +29 -0
  19. futureagi-0.5.0/fi/integrations/otel/frameworks/anthropic/examples/test-script.py +61 -0
  20. futureagi-0.5.0/fi/integrations/otel/frameworks/anthropic/version.py +1 -0
  21. futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/__generated__/__init__.py +0 -0
  22. futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/__generated__/anthropic/__init__.py +0 -0
  23. futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/__generated__/anthropic/_types.py +176 -0
  24. futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/__init__.py +491 -0
  25. futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/_proxy.py +303 -0
  26. futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/_wrappers.py +90 -0
  27. futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/examples/bedrock.sh +17 -0
  28. futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/examples/bedrock_converse.py +65 -0
  29. futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/examples/bedrock_multimodal.py +78 -0
  30. futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/examples/new-1.py +234 -0
  31. futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/examples/new-2.py +134 -0
  32. futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/package.py +2 -0
  33. futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/py.typed +0 -0
  34. futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/utils/__init__.py +93 -0
  35. futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/utils/anthropic/__init__.py +46 -0
  36. futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/utils/anthropic/_messages.py +439 -0
  37. futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/version.py +1 -0
  38. futureagi-0.5.0/fi/integrations/otel/frameworks/crewai/__init__.py +87 -0
  39. futureagi-0.5.0/fi/integrations/otel/frameworks/crewai/_wrappers.py +364 -0
  40. futureagi-0.5.0/fi/integrations/otel/frameworks/crewai/examples/crew_ai.py +154 -0
  41. futureagi-0.5.0/fi/integrations/otel/frameworks/crewai/version.py +1 -0
  42. futureagi-0.5.0/fi/integrations/otel/frameworks/dspy/__init__.py +739 -0
  43. futureagi-0.5.0/fi/integrations/otel/frameworks/dspy/examples/dspy.sh +19 -0
  44. futureagi-0.5.0/fi/integrations/otel/frameworks/dspy/examples/dspy_basic_qa.py +34 -0
  45. futureagi-0.5.0/fi/integrations/otel/frameworks/dspy/examples/lm.py +18 -0
  46. futureagi-0.5.0/fi/integrations/otel/frameworks/dspy/examples/rag_module.py +44 -0
  47. futureagi-0.5.0/fi/integrations/otel/frameworks/dspy/examples/requirements.txt +1 -0
  48. futureagi-0.5.0/fi/integrations/otel/frameworks/dspy/package.py +2 -0
  49. futureagi-0.5.0/fi/integrations/otel/frameworks/dspy/py.typed +0 -0
  50. futureagi-0.5.0/fi/integrations/otel/frameworks/dspy/version.py +1 -0
  51. futureagi-0.5.0/fi/integrations/otel/frameworks/groq/__init__.py +68 -0
  52. futureagi-0.5.0/fi/integrations/otel/frameworks/groq/_request_attributes_extractor.py +138 -0
  53. futureagi-0.5.0/fi/integrations/otel/frameworks/groq/_response_attributes_extractor.py +109 -0
  54. futureagi-0.5.0/fi/integrations/otel/frameworks/groq/_utils.py +95 -0
  55. futureagi-0.5.0/fi/integrations/otel/frameworks/groq/_with_span.py +94 -0
  56. futureagi-0.5.0/fi/integrations/otel/frameworks/groq/_wrappers.py +266 -0
  57. futureagi-0.5.0/fi/integrations/otel/frameworks/groq/examples/async_chat_completions.py +66 -0
  58. futureagi-0.5.0/fi/integrations/otel/frameworks/groq/examples/chat_completions.py +58 -0
  59. futureagi-0.5.0/fi/integrations/otel/frameworks/groq/examples/chat_completions_with_tool.py +136 -0
  60. futureagi-0.5.0/fi/integrations/otel/frameworks/groq/examples/groq.sh +19 -0
  61. futureagi-0.5.0/fi/integrations/otel/frameworks/groq/version.py +1 -0
  62. futureagi-0.5.0/fi/integrations/otel/frameworks/haystack/__init__.py +68 -0
  63. futureagi-0.5.0/fi/integrations/otel/frameworks/haystack/_wrappers.py +805 -0
  64. futureagi-0.5.0/fi/integrations/otel/frameworks/haystack/examples/building_fallbacks_with_conditional_routing.py +131 -0
  65. futureagi-0.5.0/fi/integrations/otel/frameworks/haystack/examples/cohere_reranker.py +46 -0
  66. futureagi-0.5.0/fi/integrations/otel/frameworks/haystack/examples/filtering_documents_with_metadata.py +71 -0
  67. futureagi-0.5.0/fi/integrations/otel/frameworks/haystack/examples/haystack.sh +23 -0
  68. futureagi-0.5.0/fi/integrations/otel/frameworks/haystack/examples/haystack_rag_pipeline.py +79 -0
  69. futureagi-0.5.0/fi/integrations/otel/frameworks/haystack/examples/qa_rag_pipeline.py +88 -0
  70. futureagi-0.5.0/fi/integrations/otel/frameworks/haystack/examples/requirements.txt +4 -0
  71. futureagi-0.5.0/fi/integrations/otel/frameworks/haystack/examples/tool_calling.py +81 -0
  72. futureagi-0.5.0/fi/integrations/otel/frameworks/haystack/examples/web_questions.py +51 -0
  73. futureagi-0.5.0/fi/integrations/otel/frameworks/haystack/version.py +1 -0
  74. futureagi-0.5.0/fi/integrations/otel/frameworks/instructor/__init__.py +61 -0
  75. futureagi-0.5.0/fi/integrations/otel/frameworks/instructor/_wrappers.py +249 -0
  76. futureagi-0.5.0/fi/integrations/otel/frameworks/instructor/examples/test_instructor.py +40 -0
  77. futureagi-0.5.0/fi/integrations/otel/frameworks/instructor/version.py +1 -0
  78. futureagi-0.5.0/fi/integrations/otel/frameworks/langchain/__init__.py +148 -0
  79. futureagi-0.5.0/fi/integrations/otel/frameworks/langchain/_tracer.py +1088 -0
  80. futureagi-0.5.0/fi/integrations/otel/frameworks/langchain/examples/chain_metadata.py +45 -0
  81. futureagi-0.5.0/fi/integrations/otel/frameworks/langchain/examples/chat_prompt_template.py +21 -0
  82. futureagi-0.5.0/fi/integrations/otel/frameworks/langchain/examples/custom_retriever.py +59 -0
  83. futureagi-0.5.0/fi/integrations/otel/frameworks/langchain/examples/exchange_rate_tool.py +50 -0
  84. futureagi-0.5.0/fi/integrations/otel/frameworks/langchain/examples/langchain.sh +23 -0
  85. futureagi-0.5.0/fi/integrations/otel/frameworks/langchain/examples/langgraph_agent_supervisor.py +156 -0
  86. futureagi-0.5.0/fi/integrations/otel/frameworks/langchain/examples/multimodal.py +76 -0
  87. futureagi-0.5.0/fi/integrations/otel/frameworks/langchain/examples/openai_chat_stream.py +40 -0
  88. futureagi-0.5.0/fi/integrations/otel/frameworks/langchain/examples/pathway_retriever.py +23 -0
  89. futureagi-0.5.0/fi/integrations/otel/frameworks/langchain/examples/requirements.txt +6 -0
  90. futureagi-0.5.0/fi/integrations/otel/frameworks/langchain/package.py +2 -0
  91. futureagi-0.5.0/fi/integrations/otel/frameworks/langchain/version.py +1 -0
  92. futureagi-0.5.0/fi/integrations/otel/frameworks/litellm/__init__.py +386 -0
  93. futureagi-0.5.0/fi/integrations/otel/frameworks/litellm/examples/litellm_test.py +78 -0
  94. futureagi-0.5.0/fi/integrations/otel/frameworks/litellm/package.py +2 -0
  95. futureagi-0.5.0/fi/integrations/otel/frameworks/litellm/py.typed +0 -0
  96. futureagi-0.5.0/fi/integrations/otel/frameworks/litellm/version.py +1 -0
  97. futureagi-0.5.0/fi/integrations/otel/frameworks/llama_index/__init__.py +129 -0
  98. futureagi-0.5.0/fi/integrations/otel/frameworks/llama_index/_callback.py +795 -0
  99. futureagi-0.5.0/fi/integrations/otel/frameworks/llama_index/_handler.py +1073 -0
  100. futureagi-0.5.0/fi/integrations/otel/frameworks/llama_index/examples/agent_calculator_tools.py +37 -0
  101. futureagi-0.5.0/fi/integrations/otel/frameworks/llama_index/examples/chat_engine_stream.py +38 -0
  102. futureagi-0.5.0/fi/integrations/otel/frameworks/llama_index/examples/chroma_query_engine.py +70 -0
  103. futureagi-0.5.0/fi/integrations/otel/frameworks/llama_index/examples/ingestion_pipeline.py +47 -0
  104. futureagi-0.5.0/fi/integrations/otel/frameworks/llama_index/examples/llama_index.sh +24 -0
  105. futureagi-0.5.0/fi/integrations/otel/frameworks/llama_index/examples/multimodal_stream_chat.py +62 -0
  106. futureagi-0.5.0/fi/integrations/otel/frameworks/llama_index/examples/openai_stream_chat.py +25 -0
  107. futureagi-0.5.0/fi/integrations/otel/frameworks/llama_index/examples/requirements.txt +15 -0
  108. futureagi-0.5.0/fi/integrations/otel/frameworks/llama_index/examples/text_to_sql.py +217 -0
  109. futureagi-0.5.0/fi/integrations/otel/frameworks/llama_index/examples/workflow.py +49 -0
  110. futureagi-0.5.0/fi/integrations/otel/frameworks/llama_index/package.py +2 -0
  111. futureagi-0.5.0/fi/integrations/otel/frameworks/llama_index/py.typed +0 -0
  112. futureagi-0.5.0/fi/integrations/otel/frameworks/llama_index/version.py +1 -0
  113. futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/__init__.py +139 -0
  114. futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/_chat_wrapper.py +416 -0
  115. futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/_request_attributes_extractor.py +109 -0
  116. futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/_response_accumulator.py +208 -0
  117. futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/_response_attributes_extractor.py +127 -0
  118. futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/_stream.py +183 -0
  119. futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/_utils.py +78 -0
  120. futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/_with_span.py +86 -0
  121. futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/examples/agent_completions.py +26 -0
  122. futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/examples/chat_completions.py +48 -0
  123. futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/examples/chat_completions_async.py +57 -0
  124. futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/examples/chat_completions_async_streaming.py +37 -0
  125. futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/examples/chat_completions_streaming.py +33 -0
  126. futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/examples/mistralai.sh +21 -0
  127. futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/examples/requirements.txt +1 -0
  128. futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/package.py +2 -0
  129. futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/py.typed +0 -0
  130. futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/version.py +1 -0
  131. futureagi-0.5.0/fi/integrations/otel/frameworks/openai/__init__.py +61 -0
  132. futureagi-0.5.0/fi/integrations/otel/frameworks/openai/_request.py +530 -0
  133. futureagi-0.5.0/fi/integrations/otel/frameworks/openai/_request_attributes_extractor.py +238 -0
  134. futureagi-0.5.0/fi/integrations/otel/frameworks/openai/_response_accumulator.py +275 -0
  135. futureagi-0.5.0/fi/integrations/otel/frameworks/openai/_response_attributes_extractor.py +259 -0
  136. futureagi-0.5.0/fi/integrations/otel/frameworks/openai/_span_io_handler.py +156 -0
  137. futureagi-0.5.0/fi/integrations/otel/frameworks/openai/_stream.py +204 -0
  138. futureagi-0.5.0/fi/integrations/otel/frameworks/openai/_utils.py +135 -0
  139. futureagi-0.5.0/fi/integrations/otel/frameworks/openai/_with_span.py +95 -0
  140. futureagi-0.5.0/fi/integrations/otel/frameworks/openai/examples/chat_completions.py +45 -0
  141. futureagi-0.5.0/fi/integrations/otel/frameworks/openai/examples/chat_completions_async_stream.py +57 -0
  142. futureagi-0.5.0/fi/integrations/otel/frameworks/openai/examples/chat_completions_async_stream_with_raw_response.py +36 -0
  143. futureagi-0.5.0/fi/integrations/otel/frameworks/openai/examples/chat_completions_multimodal.py +42 -0
  144. futureagi-0.5.0/fi/integrations/otel/frameworks/openai/examples/chat_completions_stream.py +28 -0
  145. futureagi-0.5.0/fi/integrations/otel/frameworks/openai/examples/chat_completions_stream_with_raw_response.py +28 -0
  146. futureagi-0.5.0/fi/integrations/otel/frameworks/openai/examples/chat_completions_with_function_calling.py +64 -0
  147. futureagi-0.5.0/fi/integrations/otel/frameworks/openai/examples/chat_completions_with_raw_response.py +24 -0
  148. futureagi-0.5.0/fi/integrations/otel/frameworks/openai/examples/chat_completions_with_streaming_response.py +28 -0
  149. futureagi-0.5.0/fi/integrations/otel/frameworks/openai/examples/embeddings.py +22 -0
  150. futureagi-0.5.0/fi/integrations/otel/frameworks/openai/examples/openai.sh +27 -0
  151. futureagi-0.5.0/fi/integrations/otel/frameworks/openai/examples/openai_tool.py +49 -0
  152. futureagi-0.5.0/fi/integrations/otel/frameworks/openai/examples/requirements.txt +1 -0
  153. futureagi-0.5.0/fi/integrations/otel/frameworks/openai/package.py +2 -0
  154. futureagi-0.5.0/fi/integrations/otel/frameworks/openai/py.typed +0 -0
  155. futureagi-0.5.0/fi/integrations/otel/frameworks/openai/test-requirements.txt +6 -0
  156. futureagi-0.5.0/fi/integrations/otel/frameworks/openai/version.py +1 -0
  157. futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/__init__.py +56 -0
  158. futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/_accumulator.py +113 -0
  159. futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/_instrumentation_status.py +7 -0
  160. futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/_proxy.py +303 -0
  161. futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/_wrapper.py +644 -0
  162. futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/examples/audio_transcription.py +30 -0
  163. futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/examples/audio_understanding.py +31 -0
  164. futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/examples/basic_generation.py +24 -0
  165. futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/examples/chat.py +22 -0
  166. futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/examples/document_understanding.py +30 -0
  167. futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/examples/function_calling.py +57 -0
  168. futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/examples/image_from_cloud_storage.py +28 -0
  169. futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/examples/requirements.txt +1 -0
  170. futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/examples/system_instructions.py +27 -0
  171. futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/examples/vertexai.model.count_tokens.py +20 -0
  172. futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/examples/vertexai.model.generate_content.py +26 -0
  173. futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/examples/vertexai.sh +28 -0
  174. futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/examples/video_and_audio_understanding.py +30 -0
  175. futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/package.py +2 -0
  176. futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/py.typed +0 -0
  177. futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/version.py +1 -0
  178. futureagi-0.5.0/fi/integrations/otel/instrumentation/.gitignore +14 -0
  179. futureagi-0.5.0/fi/integrations/otel/instrumentation/__init__.py +26 -0
  180. futureagi-0.5.0/fi/integrations/otel/instrumentation/config.py +429 -0
  181. futureagi-0.5.0/fi/integrations/otel/instrumentation/context_attributes.py +292 -0
  182. futureagi-0.5.0/fi/integrations/otel/instrumentation/helpers.py +23 -0
  183. futureagi-0.5.0/fi/integrations/otel/instrumentation/logging.py +22 -0
  184. futureagi-0.5.0/fi/integrations/otel/instrumentation/version.py +1 -0
  185. futureagi-0.5.0/fi/integrations/otel/otel.py +485 -0
  186. futureagi-0.5.0/fi/integrations/otel/settings.py +121 -0
  187. futureagi-0.5.0/fi/integrations/otel/types.py +762 -0
  188. futureagi-0.5.0/fi/test/eval_tags.py +59 -0
  189. futureagi-0.5.0/fi/test/test-3.py +208 -0
  190. {futureagi-0.4.2 → futureagi-0.5.0}/fi/utils/constants.py +4 -0
  191. {futureagi-0.4.2 → futureagi-0.5.0}/fi/utils/routes.py +1 -1
  192. futureagi-0.5.0/pyproject.toml +58 -0
  193. futureagi-0.4.2/pyproject.toml +0 -37
  194. {futureagi-0.4.2 → futureagi-0.5.0}/LICENSE.md +0 -0
  195. {futureagi-0.4.2 → futureagi-0.5.0}/README.md +0 -0
  196. {futureagi-0.4.2 → futureagi-0.5.0}/fi/__init__.py +0 -0
  197. {futureagi-0.4.2 → futureagi-0.5.0}/fi/api/__init__.py +0 -0
  198. {futureagi-0.4.2 → futureagi-0.5.0}/fi/api/apikeys.py +0 -0
  199. {futureagi-0.4.2 → futureagi-0.5.0}/fi/api/auth.py +0 -0
  200. {futureagi-0.4.2 → futureagi-0.5.0}/fi/api/types.py +0 -0
  201. {futureagi-0.4.2 → futureagi-0.5.0}/fi/client.py +0 -0
  202. {futureagi-0.4.2 → futureagi-0.5.0}/fi/datasets/__init__.py +0 -0
  203. {futureagi-0.4.2 → futureagi-0.5.0}/fi/datasets/types.py +0 -0
  204. {futureagi-0.4.2 → futureagi-0.5.0}/fi/evals/evaluator.py +0 -0
  205. {futureagi-0.4.2 → futureagi-0.5.0}/fi/evals/types.py +0 -0
  206. {futureagi-0.4.2 → futureagi-0.5.0}/fi/integrations/__init__.py +0 -0
  207. {futureagi-0.4.2 → futureagi-0.5.0}/fi/integrations/providers/__init__.py +0 -0
  208. {futureagi-0.4.2 → futureagi-0.5.0}/fi/integrations/providers/types.py +0 -0
  209. {futureagi-0.4.2 → futureagi-0.5.0}/fi/prompt/__init__.py +0 -0
  210. {futureagi-0.4.2 → futureagi-0.5.0}/fi/prompt/client.py +0 -0
  211. {futureagi-0.4.2 → futureagi-0.5.0}/fi/prompt/types.py +0 -0
  212. {futureagi-0.4.2 → futureagi-0.5.0}/fi/testcases/__init__.py +0 -0
  213. {futureagi-0.4.2 → futureagi-0.5.0}/fi/testcases/conversational_test_case.py +0 -0
  214. {futureagi-0.4.2 → futureagi-0.5.0}/fi/testcases/general.py +0 -0
  215. {futureagi-0.4.2 → futureagi-0.5.0}/fi/testcases/llm_test_case.py +0 -0
  216. {futureagi-0.4.2 → futureagi-0.5.0}/fi/testcases/mllm_test_case.py +0 -0
  217. {futureagi-0.4.2 → futureagi-0.5.0}/fi/testcases/utils.py +0 -0
  218. {futureagi-0.4.2 → futureagi-0.5.0}/fi/utils/__init__.py +0 -0
  219. {futureagi-0.4.2 → futureagi-0.5.0}/fi/utils/errors.py +0 -0
  220. {futureagi-0.4.2 → futureagi-0.5.0}/fi/utils/executor.py +0 -0
  221. {futureagi-0.4.2 → futureagi-0.5.0}/fi/utils/logging.py +0 -0
  222. {futureagi-0.4.2 → futureagi-0.5.0}/fi/utils/types.py +0 -0
  223. {futureagi-0.4.2 → futureagi-0.5.0}/fi/utils/utils.py +0 -0
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.1
1
+ Metadata-Version: 2.3
2
2
  Name: futureagi
3
- Version: 0.4.2
3
+ Version: 0.5.0
4
4
  Summary: We help GenAI teams maintain high-accuracy for their Models in production.
5
5
  Author: Future AGI
6
6
  Author-email: no-reply@futureagi.com
@@ -10,10 +10,31 @@ Classifier: Programming Language :: Python :: 3.10
10
10
  Classifier: Programming Language :: Python :: 3.11
11
11
  Classifier: Programming Language :: Python :: 3.12
12
12
  Classifier: Programming Language :: Python :: 3.13
13
+ Requires-Dist: anthropic (>=0.42.0,<0.43.0)
14
+ Requires-Dist: boto3 (>=1.28.57)
15
+ Requires-Dist: botocore (>=1.34.116) ; python_version > "3.10" and python_version < "3.13"
13
16
  Requires-Dist: certifi (==2024.6.2)
14
17
  Requires-Dist: charset-normalizer (==3.3.2)
18
+ Requires-Dist: crewai (>0.19.0) ; python_version >= "3.10" and python_version < "3.13"
19
+ Requires-Dist: dacite (>=1.9.1,<2.0.0)
20
+ Requires-Dist: dspy (>=2.5.0)
21
+ Requires-Dist: google-cloud-aiplatform (>=1.49.0) ; python_version > "3.9" and python_version < "4.0"
22
+ Requires-Dist: groq (>=0.9.0)
23
+ Requires-Dist: haystack-ai (>=2.0.0)
15
24
  Requires-Dist: idna (==3.7)
25
+ Requires-Dist: instructor (>=0.0.1) ; python_version > "3.9" and python_version < "4.0"
26
+ Requires-Dist: langchain-community[all] (>=0.3.9,<0.4.0) ; python_version >= "3.9" and python_version < "4.0"
27
+ Requires-Dist: langchain[all] (>=0.3.9,<0.4.0) ; python_version >= "3.9" and python_version < "4.0"
28
+ Requires-Dist: litellm (>=1.43.0)
29
+ Requires-Dist: llama-index-core (>=0.10.43) ; python_version > "3.9" and python_version < "4.0"
30
+ Requires-Dist: mistralai (>=1.2.6,<2.0.0) ; python_version > "3.9" and python_version < "4.0"
16
31
  Requires-Dist: numpy (==1.26.4)
32
+ Requires-Dist: openai (>=1.57.1,<2.0.0)
33
+ Requires-Dist: openinference-instrumentation-openai (>=0.1.18,<0.2.0) ; python_version >= "3.9" and python_version < "3.13"
34
+ Requires-Dist: opentelemetry-api (>=1.29.0,<2.0.0)
35
+ Requires-Dist: opentelemetry-exporter-otlp-proto-http (>=1.29.0,<2.0.0)
36
+ Requires-Dist: opentelemetry-instrumentation-openai (>=0.34.1,<0.35.0) ; python_version >= "3.9" and python_version < "4"
37
+ Requires-Dist: opentelemetry-sdk (>=1.29.0,<2.0.0)
17
38
  Requires-Dist: pandas (==2.2.2)
18
39
  Requires-Dist: pydantic (>=2.9.2,<3.0.0)
19
40
  Requires-Dist: python-dateutil (==2.9.0.post0)
@@ -21,6 +42,7 @@ Requires-Dist: pytz (==2024.1)
21
42
  Requires-Dist: requests (==2.32.3)
22
43
  Requires-Dist: requests-futures (==1.0.1)
23
44
  Requires-Dist: six (==1.16.0)
45
+ Requires-Dist: tiktoken (>=0.8.0,<0.9.0)
24
46
  Requires-Dist: tqdm (>=4.67.1,<5.0.0)
25
47
  Requires-Dist: tzdata (==2024.1)
26
48
  Requires-Dist: urllib3 (==2.2.2)
@@ -1,6 +1,6 @@
1
1
  import os
2
2
  import re
3
- from typing import List, Optional, Union, Dict, Any
3
+ from typing import Any, Dict, List, Optional, Union
4
4
 
5
5
  import pandas as pd
6
6
  from requests import Response
@@ -16,6 +16,8 @@ from fi.datasets.types import (
16
16
  HuggingfaceDatasetConfig,
17
17
  Row,
18
18
  )
19
+ from fi.evals.evaluator import EvalInfoResponseHandler
20
+ from fi.evals.templates import EvalTemplate
19
21
  from fi.utils.constants import (
20
22
  DATASET_TEMP_FILE_PREFIX,
21
23
  DATASET_TEMP_FILE_SUFFIX,
@@ -24,12 +26,9 @@ from fi.utils.constants import (
24
26
  from fi.utils.errors import InvalidAuthError, NoDatasetFound
25
27
  from fi.utils.routes import Routes
26
28
  from fi.utils.utils import get_tempfile_path
27
- from fi.evals.templates import EvalTemplate
28
- from fi.evals.evaluator import EvalInfoResponseHandler
29
29
 
30
30
 
31
31
  class DatasetResponseHandler(ResponseHandler[DatasetConfig, DatasetTable]):
32
-
33
32
  """Handles responses for dataset requests"""
34
33
 
35
34
  @classmethod
@@ -60,9 +59,11 @@ class DatasetResponseHandler(ResponseHandler[DatasetConfig, DatasetTable]):
60
59
  data_type=column["dataType"],
61
60
  source=column["originType"],
62
61
  source_id=column["sourceId"],
63
- is_frozen=column["isFrozen"]["isFrozen"]
64
- if column["isFrozen"] is not None
65
- else False,
62
+ is_frozen=(
63
+ column["isFrozen"]["isFrozen"]
64
+ if column["isFrozen"] is not None
65
+ else False
66
+ ),
66
67
  is_visible=column["isVisible"],
67
68
  eval_tags=column["evalTag"],
68
69
  average_score=column["averageScore"],
@@ -81,9 +82,11 @@ class DatasetResponseHandler(ResponseHandler[DatasetConfig, DatasetTable]):
81
82
  column_id=column_id,
82
83
  row_id=row_id,
83
84
  value=value.get("cellValue"),
84
- value_infos=[value.get("valueInfos")]
85
- if value.get("valueInfos")
86
- else None,
85
+ value_infos=(
86
+ [value.get("valueInfos")]
87
+ if value.get("valueInfos")
88
+ else None
89
+ ),
87
90
  metadata=value.get("metadata"),
88
91
  status=value.get("status"),
89
92
  failure_reason=value.get("failureReason"),
@@ -119,6 +122,7 @@ class DatasetResponseHandler(ResponseHandler[DatasetConfig, DatasetTable]):
119
122
  response.raise_for_status()
120
123
  if response.status_code == 403:
121
124
  raise InvalidAuthError()
125
+ raise Exception(response.json())
122
126
 
123
127
 
124
128
  class DatasetClient(APIKeyAuth):
@@ -134,7 +138,7 @@ class DatasetClient(APIKeyAuth):
134
138
  """
135
139
 
136
140
  _dataset_id_cache = {}
137
- __static_attributes__ = ['get_dataset_config']
141
+ __static_attributes__ = ["get_dataset_config"]
138
142
 
139
143
  def __init__(
140
144
  self,
@@ -230,29 +234,26 @@ class DatasetClient(APIKeyAuth):
230
234
 
231
235
  self._add_rows(rows)
232
236
  return self
233
-
234
- def get_column_id(self, column_name: str)-> Optional[str]:
237
+
238
+ def get_column_id(self, column_name: str) -> Optional[str]:
235
239
  if not self.dataset_config or not self.dataset_config.id:
236
240
  raise ValueError("No dataset configured")
237
-
241
+
238
242
  url = f"{self._base_url}/{Routes.dataset_table.value.format(dataset_id=str(self.dataset_config.id))}"
239
243
  dataset_table = self.request(
240
244
  config=RequestConfig(
241
- method=HttpMethod.POST,
245
+ method=HttpMethod.POST,
242
246
  url=url,
243
- json={
244
- "page_size": 1,
245
- "current_page_index": 0
246
- }
247
+ json={"page_size": 1, "current_page_index": 0},
247
248
  ),
248
- response_handler=DatasetResponseHandler
249
+ response_handler=DatasetResponseHandler,
249
250
  )
250
-
251
+
251
252
  for column in dataset_table.columns:
252
253
  if column.name == column_name:
253
254
  return str(column.id)
254
255
  return None
255
-
256
+
256
257
  def add_run_prompt(
257
258
  self,
258
259
  name: str,
@@ -267,7 +268,7 @@ class DatasetClient(APIKeyAuth):
267
268
  top_p: float = 1,
268
269
  tools: Optional[List[Dict]] = None,
269
270
  tool_choice: Optional[Any] = None,
270
- response_format: Optional[Dict] = None
271
+ response_format: Optional[Dict] = None,
271
272
  ) -> "DatasetClient":
272
273
 
273
274
  if not self.dataset_config:
@@ -275,23 +276,27 @@ class DatasetClient(APIKeyAuth):
275
276
 
276
277
  processed_messages = []
277
278
  referenced_columns = set()
278
-
279
+
279
280
  for msg in messages:
280
281
  if "role" not in msg:
281
282
  msg["role"] = "user"
282
-
283
+
283
284
  if "content" in msg:
284
285
  content = msg["content"]
285
- column_refs = re.findall(r'\{\{(.*?)\}\}', content)
286
-
286
+ column_refs = re.findall(r"\{\{(.*?)\}\}", content)
287
+
287
288
  for col_name in column_refs:
288
289
  col_id = self.get_column_id(col_name)
289
290
  if not col_id:
290
- raise ValueError(f"Referenced column '{col_name}' not found in dataset")
291
+ raise ValueError(
292
+ f"Referenced column '{col_name}' not found in dataset"
293
+ )
291
294
  referenced_columns.add(col_name)
292
- content = content.replace(f"{{{{{col_name}}}}}", f"{{{{{col_id}}}}}")
295
+ content = content.replace(
296
+ f"{{{{{col_name}}}}}", f"{{{{{col_id}}}}}"
297
+ )
293
298
  msg["content"] = content
294
-
299
+
295
300
  processed_messages.append(msg)
296
301
 
297
302
  config = {
@@ -307,7 +312,7 @@ class DatasetClient(APIKeyAuth):
307
312
  "presencePenalty": presence_penalty,
308
313
  "frequencyPenalty": frequency_penalty,
309
314
  "topP": top_p,
310
- }
315
+ },
311
316
  }
312
317
 
313
318
  if tools:
@@ -320,9 +325,9 @@ class DatasetClient(APIKeyAuth):
320
325
  url = f"{self._base_url}/{Routes.dataset_add_run_prompt_column.value}"
321
326
  self.request(
322
327
  config=RequestConfig(method=HttpMethod.POST, url=url, json=config),
323
- response_handler=DatasetResponseHandler
328
+ response_handler=DatasetResponseHandler,
324
329
  )
325
-
330
+
326
331
  return self
327
332
 
328
333
  def add_evaluation(
@@ -336,33 +341,27 @@ class DatasetClient(APIKeyAuth):
336
341
  save_as_template: bool = False,
337
342
  run: bool = True,
338
343
  reason_column: bool = False,
339
- config: Optional[Dict[str, Any]] = None
344
+ config: Optional[Dict[str, Any]] = None,
340
345
  ) -> "DatasetClient":
341
346
 
342
347
  if not self.dataset_config or not self.dataset_config.id:
343
348
  raise ValueError("No dataset configured")
344
349
 
345
- template_classes = {
346
- cls.__name__: cls
347
- for cls in EvalTemplate.__subclasses__()
348
- }
350
+ template_classes = {cls.__name__: cls for cls in EvalTemplate.__subclasses__()}
349
351
  if eval_template not in template_classes:
350
352
  raise ValueError(f"Unknown template name: {eval_template}")
351
-
353
+
352
354
  eval_id = template_classes[eval_template].eval_id
353
355
 
354
356
  url = f"{self._base_url}/sdk/api/v1/eval/{eval_id}/"
355
357
  template_details = self.request(
356
- config=RequestConfig(
357
- method=HttpMethod.GET,
358
- url=url
359
- ),
360
- response_handler=EvalInfoResponseHandler
358
+ config=RequestConfig(method=HttpMethod.GET, url=url),
359
+ response_handler=EvalInfoResponseHandler,
361
360
  )
362
-
361
+
363
362
  template_id = template_details["id"]
364
363
  required_keys = template_details["config"]["required_keys"]
365
-
364
+
366
365
  mapping = {}
367
366
 
368
367
  if "input" in required_keys:
@@ -372,7 +371,7 @@ class DatasetClient(APIKeyAuth):
372
371
  if not input_column_id:
373
372
  raise ValueError(f"Column '{input_column_name}' not found in dataset")
374
373
  mapping["input"] = input_column_id
375
-
374
+
376
375
  if "output" in required_keys or "response" in required_keys:
377
376
  if not output_column_name:
378
377
  raise ValueError(f"Output column required for {eval_template}")
@@ -383,7 +382,7 @@ class DatasetClient(APIKeyAuth):
383
382
  mapping["output"] = output_column_id
384
383
  else:
385
384
  mapping["response"] = output_column_id
386
-
385
+
387
386
  if "context" in required_keys:
388
387
  if not context_column_name:
389
388
  raise ValueError(f"Context column required for {eval_template}")
@@ -391,13 +390,17 @@ class DatasetClient(APIKeyAuth):
391
390
  if not context_column_id:
392
391
  raise ValueError(f"Column '{context_column_name}' not found in dataset")
393
392
  mapping["context"] = context_column_id
394
-
393
+
395
394
  if "expected_response" in required_keys:
396
395
  if not expected_column_name:
397
- raise ValueError(f"Expected response column required for {eval_template}")
396
+ raise ValueError(
397
+ f"Expected response column required for {eval_template}"
398
+ )
398
399
  expected_column_id = self.get_column_id(expected_column_name)
399
400
  if not expected_column_id:
400
- raise ValueError(f"Column '{expected_column_name}' not found in dataset")
401
+ raise ValueError(
402
+ f"Column '{expected_column_name}' not found in dataset"
403
+ )
401
404
  mapping["expected_response"] = expected_column_id
402
405
 
403
406
  eval_config = {
@@ -408,20 +411,16 @@ class DatasetClient(APIKeyAuth):
408
411
  "config": {
409
412
  "mapping": mapping,
410
413
  "config": config or {},
411
- "reasonColumn": reason_column
412
- }
414
+ "reasonColumn": reason_column,
415
+ },
413
416
  }
414
417
 
415
418
  url = f"{self._base_url}/model-hub/develops/{str(self.dataset_config.id)}/add_user_eval/"
416
419
  self.request(
417
- config=RequestConfig(
418
- method=HttpMethod.POST,
419
- url=url,
420
- json=eval_config
421
- ),
422
- response_handler=DatasetResponseHandler
420
+ config=RequestConfig(method=HttpMethod.POST, url=url, json=eval_config),
421
+ response_handler=DatasetResponseHandler,
423
422
  )
424
-
423
+
425
424
  return self
426
425
 
427
426
  def add_optimization(
@@ -429,7 +428,7 @@ class DatasetClient(APIKeyAuth):
429
428
  optimization_name: str,
430
429
  prompt_column_name: str,
431
430
  optimize_type: str = "PROMPT_TEMPLATE",
432
- model_config: Optional[Dict[str, Any]] = None
431
+ model_config: Optional[Dict[str, Any]] = None,
433
432
  ) -> "DatasetClient":
434
433
 
435
434
  if not self.dataset_config or not self.dataset_config.id:
@@ -437,17 +436,17 @@ class DatasetClient(APIKeyAuth):
437
436
 
438
437
  column_id = self.get_column_id(prompt_column_name)
439
438
  if not column_id:
440
- raise ValueError(f"Prompt Column '{prompt_column_name}' not found in dataset")
439
+ raise ValueError(
440
+ f"Prompt Column '{prompt_column_name}' not found in dataset"
441
+ )
441
442
 
442
- #requires mapping input for userEvalTemplateIds
443
+ # requires mapping input for userEvalTemplateIds
443
444
  metrics_url = f"{self._base_url}/model-hub/metrics/by-column/"
444
445
  metrics_response = self.request(
445
446
  config=RequestConfig(
446
- method=HttpMethod.GET,
447
- url=metrics_url,
448
- params={"column_id": column_id}
447
+ method=HttpMethod.GET, url=metrics_url, params={"column_id": column_id}
449
448
  ),
450
- response_handler=DatasetResponseHandler
449
+ response_handler=DatasetResponseHandler,
451
450
  )
452
451
 
453
452
  eval_template_ids = []
@@ -456,7 +455,9 @@ class DatasetClient(APIKeyAuth):
456
455
  eval_template_ids.append(metric["id"])
457
456
 
458
457
  if not eval_template_ids:
459
- raise ValueError(f"No evaluation templates found for column '{prompt_column_name}'")
458
+ raise ValueError(
459
+ f"No evaluation templates found for column '{prompt_column_name}'"
460
+ )
460
461
 
461
462
  payload = {
462
463
  "name": optimization_name,
@@ -464,19 +465,15 @@ class DatasetClient(APIKeyAuth):
464
465
  "optimizeType": optimize_type,
465
466
  "userEvalTemplateIds": eval_template_ids,
466
467
  "datasetId": str(self.dataset_config.id),
467
- "modelConfig": model_config or {}
468
+ "modelConfig": model_config or {},
468
469
  }
469
470
 
470
471
  opt_url = f"{self._base_url}/model-hub/optimisation/create/"
471
472
  self.request(
472
- config=RequestConfig(
473
- method=HttpMethod.POST,
474
- url=opt_url,
475
- json=payload
476
- ),
477
- response_handler=DatasetResponseHandler
473
+ config=RequestConfig(method=HttpMethod.POST, url=opt_url, json=payload),
474
+ response_handler=DatasetResponseHandler,
478
475
  )
479
-
476
+
480
477
  return self
481
478
 
482
479
  # Protected internal methods
@@ -1,4 +1,5 @@
1
1
  from fi.evals.evaluator import EvalClient # noqa: F401
2
+ from fi.evals.protect import ProtectClient # noqa: F401
2
3
  from fi.evals.templates import (
3
4
  PII,
4
5
  AgentJudge,
@@ -130,4 +131,5 @@ __all__ = [
130
131
  "Toxicity",
131
132
  "TranslationAccuracy",
132
133
  "EvalClient",
134
+ "ProtectClient",
133
135
  ]