futureagi 0.4.2__tar.gz → 0.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of futureagi might be problematic. Click here for more details.
- {futureagi-0.4.2 → futureagi-0.5.0}/PKG-INFO +24 -2
- {futureagi-0.4.2 → futureagi-0.5.0}/fi/datasets/client.py +73 -76
- {futureagi-0.4.2 → futureagi-0.5.0}/fi/evals/__init__.py +2 -0
- futureagi-0.5.0/fi/evals/protect.py +409 -0
- {futureagi-0.4.2 → futureagi-0.5.0}/fi/evals/templates.py +16 -0
- futureagi-0.5.0/fi/integrations/otel/__init__.py +50 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/anthropic/__init__.py +117 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/anthropic/_stream.py +556 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/anthropic/_utils.py +108 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/anthropic/_with_span.py +119 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/anthropic/_wrappers.py +856 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/anthropic/examples/anthropic.sh +21 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/anthropic/examples/end_to_end_tool_use.py +125 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/anthropic/examples/multiple_tool_calling.py +65 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/anthropic/examples/new-1.py +238 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/anthropic/examples/requirements.txt +1 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/anthropic/examples/sync_completions.py +32 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/anthropic/examples/sync_messages.py +29 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/anthropic/examples/test-script.py +61 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/anthropic/version.py +1 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/__generated__/__init__.py +0 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/__generated__/anthropic/__init__.py +0 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/__generated__/anthropic/_types.py +176 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/__init__.py +491 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/_proxy.py +303 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/_wrappers.py +90 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/examples/bedrock.sh +17 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/examples/bedrock_converse.py +65 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/examples/bedrock_multimodal.py +78 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/examples/new-1.py +234 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/examples/new-2.py +134 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/package.py +2 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/py.typed +0 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/utils/__init__.py +93 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/utils/anthropic/__init__.py +46 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/utils/anthropic/_messages.py +439 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/version.py +1 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/crewai/__init__.py +87 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/crewai/_wrappers.py +364 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/crewai/examples/crew_ai.py +154 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/crewai/version.py +1 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/dspy/__init__.py +739 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/dspy/examples/dspy.sh +19 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/dspy/examples/dspy_basic_qa.py +34 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/dspy/examples/lm.py +18 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/dspy/examples/rag_module.py +44 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/dspy/examples/requirements.txt +1 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/dspy/package.py +2 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/dspy/py.typed +0 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/dspy/version.py +1 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/groq/__init__.py +68 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/groq/_request_attributes_extractor.py +138 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/groq/_response_attributes_extractor.py +109 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/groq/_utils.py +95 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/groq/_with_span.py +94 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/groq/_wrappers.py +266 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/groq/examples/async_chat_completions.py +66 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/groq/examples/chat_completions.py +58 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/groq/examples/chat_completions_with_tool.py +136 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/groq/examples/groq.sh +19 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/groq/version.py +1 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/haystack/__init__.py +68 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/haystack/_wrappers.py +805 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/haystack/examples/building_fallbacks_with_conditional_routing.py +131 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/haystack/examples/cohere_reranker.py +46 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/haystack/examples/filtering_documents_with_metadata.py +71 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/haystack/examples/haystack.sh +23 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/haystack/examples/haystack_rag_pipeline.py +79 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/haystack/examples/qa_rag_pipeline.py +88 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/haystack/examples/requirements.txt +4 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/haystack/examples/tool_calling.py +81 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/haystack/examples/web_questions.py +51 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/haystack/version.py +1 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/instructor/__init__.py +61 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/instructor/_wrappers.py +249 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/instructor/examples/test_instructor.py +40 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/instructor/version.py +1 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/langchain/__init__.py +148 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/langchain/_tracer.py +1088 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/langchain/examples/chain_metadata.py +45 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/langchain/examples/chat_prompt_template.py +21 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/langchain/examples/custom_retriever.py +59 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/langchain/examples/exchange_rate_tool.py +50 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/langchain/examples/langchain.sh +23 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/langchain/examples/langgraph_agent_supervisor.py +156 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/langchain/examples/multimodal.py +76 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/langchain/examples/openai_chat_stream.py +40 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/langchain/examples/pathway_retriever.py +23 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/langchain/examples/requirements.txt +6 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/langchain/package.py +2 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/langchain/version.py +1 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/litellm/__init__.py +386 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/litellm/examples/litellm_test.py +78 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/litellm/package.py +2 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/litellm/py.typed +0 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/litellm/version.py +1 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/llama_index/__init__.py +129 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/llama_index/_callback.py +795 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/llama_index/_handler.py +1073 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/llama_index/examples/agent_calculator_tools.py +37 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/llama_index/examples/chat_engine_stream.py +38 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/llama_index/examples/chroma_query_engine.py +70 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/llama_index/examples/ingestion_pipeline.py +47 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/llama_index/examples/llama_index.sh +24 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/llama_index/examples/multimodal_stream_chat.py +62 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/llama_index/examples/openai_stream_chat.py +25 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/llama_index/examples/requirements.txt +15 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/llama_index/examples/text_to_sql.py +217 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/llama_index/examples/workflow.py +49 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/llama_index/package.py +2 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/llama_index/py.typed +0 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/llama_index/version.py +1 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/__init__.py +139 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/_chat_wrapper.py +416 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/_request_attributes_extractor.py +109 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/_response_accumulator.py +208 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/_response_attributes_extractor.py +127 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/_stream.py +183 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/_utils.py +78 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/_with_span.py +86 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/examples/agent_completions.py +26 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/examples/chat_completions.py +48 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/examples/chat_completions_async.py +57 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/examples/chat_completions_async_streaming.py +37 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/examples/chat_completions_streaming.py +33 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/examples/mistralai.sh +21 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/examples/requirements.txt +1 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/package.py +2 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/py.typed +0 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/version.py +1 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/__init__.py +61 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/_request.py +530 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/_request_attributes_extractor.py +238 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/_response_accumulator.py +275 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/_response_attributes_extractor.py +259 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/_span_io_handler.py +156 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/_stream.py +204 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/_utils.py +135 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/_with_span.py +95 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/examples/chat_completions.py +45 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/examples/chat_completions_async_stream.py +57 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/examples/chat_completions_async_stream_with_raw_response.py +36 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/examples/chat_completions_multimodal.py +42 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/examples/chat_completions_stream.py +28 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/examples/chat_completions_stream_with_raw_response.py +28 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/examples/chat_completions_with_function_calling.py +64 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/examples/chat_completions_with_raw_response.py +24 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/examples/chat_completions_with_streaming_response.py +28 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/examples/embeddings.py +22 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/examples/openai.sh +27 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/examples/openai_tool.py +49 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/examples/requirements.txt +1 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/package.py +2 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/py.typed +0 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/test-requirements.txt +6 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/version.py +1 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/__init__.py +56 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/_accumulator.py +113 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/_instrumentation_status.py +7 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/_proxy.py +303 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/_wrapper.py +644 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/examples/audio_transcription.py +30 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/examples/audio_understanding.py +31 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/examples/basic_generation.py +24 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/examples/chat.py +22 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/examples/document_understanding.py +30 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/examples/function_calling.py +57 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/examples/image_from_cloud_storage.py +28 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/examples/requirements.txt +1 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/examples/system_instructions.py +27 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/examples/vertexai.model.count_tokens.py +20 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/examples/vertexai.model.generate_content.py +26 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/examples/vertexai.sh +28 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/examples/video_and_audio_understanding.py +30 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/package.py +2 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/py.typed +0 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/version.py +1 -0
- futureagi-0.5.0/fi/integrations/otel/instrumentation/.gitignore +14 -0
- futureagi-0.5.0/fi/integrations/otel/instrumentation/__init__.py +26 -0
- futureagi-0.5.0/fi/integrations/otel/instrumentation/config.py +429 -0
- futureagi-0.5.0/fi/integrations/otel/instrumentation/context_attributes.py +292 -0
- futureagi-0.5.0/fi/integrations/otel/instrumentation/helpers.py +23 -0
- futureagi-0.5.0/fi/integrations/otel/instrumentation/logging.py +22 -0
- futureagi-0.5.0/fi/integrations/otel/instrumentation/version.py +1 -0
- futureagi-0.5.0/fi/integrations/otel/otel.py +485 -0
- futureagi-0.5.0/fi/integrations/otel/settings.py +121 -0
- futureagi-0.5.0/fi/integrations/otel/types.py +762 -0
- futureagi-0.5.0/fi/test/eval_tags.py +59 -0
- futureagi-0.5.0/fi/test/test-3.py +208 -0
- {futureagi-0.4.2 → futureagi-0.5.0}/fi/utils/constants.py +4 -0
- {futureagi-0.4.2 → futureagi-0.5.0}/fi/utils/routes.py +1 -1
- futureagi-0.5.0/pyproject.toml +58 -0
- futureagi-0.4.2/pyproject.toml +0 -37
- {futureagi-0.4.2 → futureagi-0.5.0}/LICENSE.md +0 -0
- {futureagi-0.4.2 → futureagi-0.5.0}/README.md +0 -0
- {futureagi-0.4.2 → futureagi-0.5.0}/fi/__init__.py +0 -0
- {futureagi-0.4.2 → futureagi-0.5.0}/fi/api/__init__.py +0 -0
- {futureagi-0.4.2 → futureagi-0.5.0}/fi/api/apikeys.py +0 -0
- {futureagi-0.4.2 → futureagi-0.5.0}/fi/api/auth.py +0 -0
- {futureagi-0.4.2 → futureagi-0.5.0}/fi/api/types.py +0 -0
- {futureagi-0.4.2 → futureagi-0.5.0}/fi/client.py +0 -0
- {futureagi-0.4.2 → futureagi-0.5.0}/fi/datasets/__init__.py +0 -0
- {futureagi-0.4.2 → futureagi-0.5.0}/fi/datasets/types.py +0 -0
- {futureagi-0.4.2 → futureagi-0.5.0}/fi/evals/evaluator.py +0 -0
- {futureagi-0.4.2 → futureagi-0.5.0}/fi/evals/types.py +0 -0
- {futureagi-0.4.2 → futureagi-0.5.0}/fi/integrations/__init__.py +0 -0
- {futureagi-0.4.2 → futureagi-0.5.0}/fi/integrations/providers/__init__.py +0 -0
- {futureagi-0.4.2 → futureagi-0.5.0}/fi/integrations/providers/types.py +0 -0
- {futureagi-0.4.2 → futureagi-0.5.0}/fi/prompt/__init__.py +0 -0
- {futureagi-0.4.2 → futureagi-0.5.0}/fi/prompt/client.py +0 -0
- {futureagi-0.4.2 → futureagi-0.5.0}/fi/prompt/types.py +0 -0
- {futureagi-0.4.2 → futureagi-0.5.0}/fi/testcases/__init__.py +0 -0
- {futureagi-0.4.2 → futureagi-0.5.0}/fi/testcases/conversational_test_case.py +0 -0
- {futureagi-0.4.2 → futureagi-0.5.0}/fi/testcases/general.py +0 -0
- {futureagi-0.4.2 → futureagi-0.5.0}/fi/testcases/llm_test_case.py +0 -0
- {futureagi-0.4.2 → futureagi-0.5.0}/fi/testcases/mllm_test_case.py +0 -0
- {futureagi-0.4.2 → futureagi-0.5.0}/fi/testcases/utils.py +0 -0
- {futureagi-0.4.2 → futureagi-0.5.0}/fi/utils/__init__.py +0 -0
- {futureagi-0.4.2 → futureagi-0.5.0}/fi/utils/errors.py +0 -0
- {futureagi-0.4.2 → futureagi-0.5.0}/fi/utils/executor.py +0 -0
- {futureagi-0.4.2 → futureagi-0.5.0}/fi/utils/logging.py +0 -0
- {futureagi-0.4.2 → futureagi-0.5.0}/fi/utils/types.py +0 -0
- {futureagi-0.4.2 → futureagi-0.5.0}/fi/utils/utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
2
|
Name: futureagi
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.5.0
|
|
4
4
|
Summary: We help GenAI teams maintain high-accuracy for their Models in production.
|
|
5
5
|
Author: Future AGI
|
|
6
6
|
Author-email: no-reply@futureagi.com
|
|
@@ -10,10 +10,31 @@ Classifier: Programming Language :: Python :: 3.10
|
|
|
10
10
|
Classifier: Programming Language :: Python :: 3.11
|
|
11
11
|
Classifier: Programming Language :: Python :: 3.12
|
|
12
12
|
Classifier: Programming Language :: Python :: 3.13
|
|
13
|
+
Requires-Dist: anthropic (>=0.42.0,<0.43.0)
|
|
14
|
+
Requires-Dist: boto3 (>=1.28.57)
|
|
15
|
+
Requires-Dist: botocore (>=1.34.116) ; python_version > "3.10" and python_version < "3.13"
|
|
13
16
|
Requires-Dist: certifi (==2024.6.2)
|
|
14
17
|
Requires-Dist: charset-normalizer (==3.3.2)
|
|
18
|
+
Requires-Dist: crewai (>0.19.0) ; python_version >= "3.10" and python_version < "3.13"
|
|
19
|
+
Requires-Dist: dacite (>=1.9.1,<2.0.0)
|
|
20
|
+
Requires-Dist: dspy (>=2.5.0)
|
|
21
|
+
Requires-Dist: google-cloud-aiplatform (>=1.49.0) ; python_version > "3.9" and python_version < "4.0"
|
|
22
|
+
Requires-Dist: groq (>=0.9.0)
|
|
23
|
+
Requires-Dist: haystack-ai (>=2.0.0)
|
|
15
24
|
Requires-Dist: idna (==3.7)
|
|
25
|
+
Requires-Dist: instructor (>=0.0.1) ; python_version > "3.9" and python_version < "4.0"
|
|
26
|
+
Requires-Dist: langchain-community[all] (>=0.3.9,<0.4.0) ; python_version >= "3.9" and python_version < "4.0"
|
|
27
|
+
Requires-Dist: langchain[all] (>=0.3.9,<0.4.0) ; python_version >= "3.9" and python_version < "4.0"
|
|
28
|
+
Requires-Dist: litellm (>=1.43.0)
|
|
29
|
+
Requires-Dist: llama-index-core (>=0.10.43) ; python_version > "3.9" and python_version < "4.0"
|
|
30
|
+
Requires-Dist: mistralai (>=1.2.6,<2.0.0) ; python_version > "3.9" and python_version < "4.0"
|
|
16
31
|
Requires-Dist: numpy (==1.26.4)
|
|
32
|
+
Requires-Dist: openai (>=1.57.1,<2.0.0)
|
|
33
|
+
Requires-Dist: openinference-instrumentation-openai (>=0.1.18,<0.2.0) ; python_version >= "3.9" and python_version < "3.13"
|
|
34
|
+
Requires-Dist: opentelemetry-api (>=1.29.0,<2.0.0)
|
|
35
|
+
Requires-Dist: opentelemetry-exporter-otlp-proto-http (>=1.29.0,<2.0.0)
|
|
36
|
+
Requires-Dist: opentelemetry-instrumentation-openai (>=0.34.1,<0.35.0) ; python_version >= "3.9" and python_version < "4"
|
|
37
|
+
Requires-Dist: opentelemetry-sdk (>=1.29.0,<2.0.0)
|
|
17
38
|
Requires-Dist: pandas (==2.2.2)
|
|
18
39
|
Requires-Dist: pydantic (>=2.9.2,<3.0.0)
|
|
19
40
|
Requires-Dist: python-dateutil (==2.9.0.post0)
|
|
@@ -21,6 +42,7 @@ Requires-Dist: pytz (==2024.1)
|
|
|
21
42
|
Requires-Dist: requests (==2.32.3)
|
|
22
43
|
Requires-Dist: requests-futures (==1.0.1)
|
|
23
44
|
Requires-Dist: six (==1.16.0)
|
|
45
|
+
Requires-Dist: tiktoken (>=0.8.0,<0.9.0)
|
|
24
46
|
Requires-Dist: tqdm (>=4.67.1,<5.0.0)
|
|
25
47
|
Requires-Dist: tzdata (==2024.1)
|
|
26
48
|
Requires-Dist: urllib3 (==2.2.2)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import os
|
|
2
2
|
import re
|
|
3
|
-
from typing import
|
|
3
|
+
from typing import Any, Dict, List, Optional, Union
|
|
4
4
|
|
|
5
5
|
import pandas as pd
|
|
6
6
|
from requests import Response
|
|
@@ -16,6 +16,8 @@ from fi.datasets.types import (
|
|
|
16
16
|
HuggingfaceDatasetConfig,
|
|
17
17
|
Row,
|
|
18
18
|
)
|
|
19
|
+
from fi.evals.evaluator import EvalInfoResponseHandler
|
|
20
|
+
from fi.evals.templates import EvalTemplate
|
|
19
21
|
from fi.utils.constants import (
|
|
20
22
|
DATASET_TEMP_FILE_PREFIX,
|
|
21
23
|
DATASET_TEMP_FILE_SUFFIX,
|
|
@@ -24,12 +26,9 @@ from fi.utils.constants import (
|
|
|
24
26
|
from fi.utils.errors import InvalidAuthError, NoDatasetFound
|
|
25
27
|
from fi.utils.routes import Routes
|
|
26
28
|
from fi.utils.utils import get_tempfile_path
|
|
27
|
-
from fi.evals.templates import EvalTemplate
|
|
28
|
-
from fi.evals.evaluator import EvalInfoResponseHandler
|
|
29
29
|
|
|
30
30
|
|
|
31
31
|
class DatasetResponseHandler(ResponseHandler[DatasetConfig, DatasetTable]):
|
|
32
|
-
|
|
33
32
|
"""Handles responses for dataset requests"""
|
|
34
33
|
|
|
35
34
|
@classmethod
|
|
@@ -60,9 +59,11 @@ class DatasetResponseHandler(ResponseHandler[DatasetConfig, DatasetTable]):
|
|
|
60
59
|
data_type=column["dataType"],
|
|
61
60
|
source=column["originType"],
|
|
62
61
|
source_id=column["sourceId"],
|
|
63
|
-
is_frozen=
|
|
64
|
-
|
|
65
|
-
|
|
62
|
+
is_frozen=(
|
|
63
|
+
column["isFrozen"]["isFrozen"]
|
|
64
|
+
if column["isFrozen"] is not None
|
|
65
|
+
else False
|
|
66
|
+
),
|
|
66
67
|
is_visible=column["isVisible"],
|
|
67
68
|
eval_tags=column["evalTag"],
|
|
68
69
|
average_score=column["averageScore"],
|
|
@@ -81,9 +82,11 @@ class DatasetResponseHandler(ResponseHandler[DatasetConfig, DatasetTable]):
|
|
|
81
82
|
column_id=column_id,
|
|
82
83
|
row_id=row_id,
|
|
83
84
|
value=value.get("cellValue"),
|
|
84
|
-
value_infos=
|
|
85
|
-
|
|
86
|
-
|
|
85
|
+
value_infos=(
|
|
86
|
+
[value.get("valueInfos")]
|
|
87
|
+
if value.get("valueInfos")
|
|
88
|
+
else None
|
|
89
|
+
),
|
|
87
90
|
metadata=value.get("metadata"),
|
|
88
91
|
status=value.get("status"),
|
|
89
92
|
failure_reason=value.get("failureReason"),
|
|
@@ -119,6 +122,7 @@ class DatasetResponseHandler(ResponseHandler[DatasetConfig, DatasetTable]):
|
|
|
119
122
|
response.raise_for_status()
|
|
120
123
|
if response.status_code == 403:
|
|
121
124
|
raise InvalidAuthError()
|
|
125
|
+
raise Exception(response.json())
|
|
122
126
|
|
|
123
127
|
|
|
124
128
|
class DatasetClient(APIKeyAuth):
|
|
@@ -134,7 +138,7 @@ class DatasetClient(APIKeyAuth):
|
|
|
134
138
|
"""
|
|
135
139
|
|
|
136
140
|
_dataset_id_cache = {}
|
|
137
|
-
__static_attributes__ = [
|
|
141
|
+
__static_attributes__ = ["get_dataset_config"]
|
|
138
142
|
|
|
139
143
|
def __init__(
|
|
140
144
|
self,
|
|
@@ -230,29 +234,26 @@ class DatasetClient(APIKeyAuth):
|
|
|
230
234
|
|
|
231
235
|
self._add_rows(rows)
|
|
232
236
|
return self
|
|
233
|
-
|
|
234
|
-
def get_column_id(self, column_name: str)-> Optional[str]:
|
|
237
|
+
|
|
238
|
+
def get_column_id(self, column_name: str) -> Optional[str]:
|
|
235
239
|
if not self.dataset_config or not self.dataset_config.id:
|
|
236
240
|
raise ValueError("No dataset configured")
|
|
237
|
-
|
|
241
|
+
|
|
238
242
|
url = f"{self._base_url}/{Routes.dataset_table.value.format(dataset_id=str(self.dataset_config.id))}"
|
|
239
243
|
dataset_table = self.request(
|
|
240
244
|
config=RequestConfig(
|
|
241
|
-
method=HttpMethod.POST,
|
|
245
|
+
method=HttpMethod.POST,
|
|
242
246
|
url=url,
|
|
243
|
-
json={
|
|
244
|
-
"page_size": 1,
|
|
245
|
-
"current_page_index": 0
|
|
246
|
-
}
|
|
247
|
+
json={"page_size": 1, "current_page_index": 0},
|
|
247
248
|
),
|
|
248
|
-
response_handler=DatasetResponseHandler
|
|
249
|
+
response_handler=DatasetResponseHandler,
|
|
249
250
|
)
|
|
250
|
-
|
|
251
|
+
|
|
251
252
|
for column in dataset_table.columns:
|
|
252
253
|
if column.name == column_name:
|
|
253
254
|
return str(column.id)
|
|
254
255
|
return None
|
|
255
|
-
|
|
256
|
+
|
|
256
257
|
def add_run_prompt(
|
|
257
258
|
self,
|
|
258
259
|
name: str,
|
|
@@ -267,7 +268,7 @@ class DatasetClient(APIKeyAuth):
|
|
|
267
268
|
top_p: float = 1,
|
|
268
269
|
tools: Optional[List[Dict]] = None,
|
|
269
270
|
tool_choice: Optional[Any] = None,
|
|
270
|
-
response_format: Optional[Dict] = None
|
|
271
|
+
response_format: Optional[Dict] = None,
|
|
271
272
|
) -> "DatasetClient":
|
|
272
273
|
|
|
273
274
|
if not self.dataset_config:
|
|
@@ -275,23 +276,27 @@ class DatasetClient(APIKeyAuth):
|
|
|
275
276
|
|
|
276
277
|
processed_messages = []
|
|
277
278
|
referenced_columns = set()
|
|
278
|
-
|
|
279
|
+
|
|
279
280
|
for msg in messages:
|
|
280
281
|
if "role" not in msg:
|
|
281
282
|
msg["role"] = "user"
|
|
282
|
-
|
|
283
|
+
|
|
283
284
|
if "content" in msg:
|
|
284
285
|
content = msg["content"]
|
|
285
|
-
column_refs = re.findall(r
|
|
286
|
-
|
|
286
|
+
column_refs = re.findall(r"\{\{(.*?)\}\}", content)
|
|
287
|
+
|
|
287
288
|
for col_name in column_refs:
|
|
288
289
|
col_id = self.get_column_id(col_name)
|
|
289
290
|
if not col_id:
|
|
290
|
-
raise ValueError(
|
|
291
|
+
raise ValueError(
|
|
292
|
+
f"Referenced column '{col_name}' not found in dataset"
|
|
293
|
+
)
|
|
291
294
|
referenced_columns.add(col_name)
|
|
292
|
-
content = content.replace(
|
|
295
|
+
content = content.replace(
|
|
296
|
+
f"{{{{{col_name}}}}}", f"{{{{{col_id}}}}}"
|
|
297
|
+
)
|
|
293
298
|
msg["content"] = content
|
|
294
|
-
|
|
299
|
+
|
|
295
300
|
processed_messages.append(msg)
|
|
296
301
|
|
|
297
302
|
config = {
|
|
@@ -307,7 +312,7 @@ class DatasetClient(APIKeyAuth):
|
|
|
307
312
|
"presencePenalty": presence_penalty,
|
|
308
313
|
"frequencyPenalty": frequency_penalty,
|
|
309
314
|
"topP": top_p,
|
|
310
|
-
}
|
|
315
|
+
},
|
|
311
316
|
}
|
|
312
317
|
|
|
313
318
|
if tools:
|
|
@@ -320,9 +325,9 @@ class DatasetClient(APIKeyAuth):
|
|
|
320
325
|
url = f"{self._base_url}/{Routes.dataset_add_run_prompt_column.value}"
|
|
321
326
|
self.request(
|
|
322
327
|
config=RequestConfig(method=HttpMethod.POST, url=url, json=config),
|
|
323
|
-
response_handler=DatasetResponseHandler
|
|
328
|
+
response_handler=DatasetResponseHandler,
|
|
324
329
|
)
|
|
325
|
-
|
|
330
|
+
|
|
326
331
|
return self
|
|
327
332
|
|
|
328
333
|
def add_evaluation(
|
|
@@ -336,33 +341,27 @@ class DatasetClient(APIKeyAuth):
|
|
|
336
341
|
save_as_template: bool = False,
|
|
337
342
|
run: bool = True,
|
|
338
343
|
reason_column: bool = False,
|
|
339
|
-
config: Optional[Dict[str, Any]] = None
|
|
344
|
+
config: Optional[Dict[str, Any]] = None,
|
|
340
345
|
) -> "DatasetClient":
|
|
341
346
|
|
|
342
347
|
if not self.dataset_config or not self.dataset_config.id:
|
|
343
348
|
raise ValueError("No dataset configured")
|
|
344
349
|
|
|
345
|
-
template_classes = {
|
|
346
|
-
cls.__name__: cls
|
|
347
|
-
for cls in EvalTemplate.__subclasses__()
|
|
348
|
-
}
|
|
350
|
+
template_classes = {cls.__name__: cls for cls in EvalTemplate.__subclasses__()}
|
|
349
351
|
if eval_template not in template_classes:
|
|
350
352
|
raise ValueError(f"Unknown template name: {eval_template}")
|
|
351
|
-
|
|
353
|
+
|
|
352
354
|
eval_id = template_classes[eval_template].eval_id
|
|
353
355
|
|
|
354
356
|
url = f"{self._base_url}/sdk/api/v1/eval/{eval_id}/"
|
|
355
357
|
template_details = self.request(
|
|
356
|
-
config=RequestConfig(
|
|
357
|
-
|
|
358
|
-
url=url
|
|
359
|
-
),
|
|
360
|
-
response_handler=EvalInfoResponseHandler
|
|
358
|
+
config=RequestConfig(method=HttpMethod.GET, url=url),
|
|
359
|
+
response_handler=EvalInfoResponseHandler,
|
|
361
360
|
)
|
|
362
|
-
|
|
361
|
+
|
|
363
362
|
template_id = template_details["id"]
|
|
364
363
|
required_keys = template_details["config"]["required_keys"]
|
|
365
|
-
|
|
364
|
+
|
|
366
365
|
mapping = {}
|
|
367
366
|
|
|
368
367
|
if "input" in required_keys:
|
|
@@ -372,7 +371,7 @@ class DatasetClient(APIKeyAuth):
|
|
|
372
371
|
if not input_column_id:
|
|
373
372
|
raise ValueError(f"Column '{input_column_name}' not found in dataset")
|
|
374
373
|
mapping["input"] = input_column_id
|
|
375
|
-
|
|
374
|
+
|
|
376
375
|
if "output" in required_keys or "response" in required_keys:
|
|
377
376
|
if not output_column_name:
|
|
378
377
|
raise ValueError(f"Output column required for {eval_template}")
|
|
@@ -383,7 +382,7 @@ class DatasetClient(APIKeyAuth):
|
|
|
383
382
|
mapping["output"] = output_column_id
|
|
384
383
|
else:
|
|
385
384
|
mapping["response"] = output_column_id
|
|
386
|
-
|
|
385
|
+
|
|
387
386
|
if "context" in required_keys:
|
|
388
387
|
if not context_column_name:
|
|
389
388
|
raise ValueError(f"Context column required for {eval_template}")
|
|
@@ -391,13 +390,17 @@ class DatasetClient(APIKeyAuth):
|
|
|
391
390
|
if not context_column_id:
|
|
392
391
|
raise ValueError(f"Column '{context_column_name}' not found in dataset")
|
|
393
392
|
mapping["context"] = context_column_id
|
|
394
|
-
|
|
393
|
+
|
|
395
394
|
if "expected_response" in required_keys:
|
|
396
395
|
if not expected_column_name:
|
|
397
|
-
raise ValueError(
|
|
396
|
+
raise ValueError(
|
|
397
|
+
f"Expected response column required for {eval_template}"
|
|
398
|
+
)
|
|
398
399
|
expected_column_id = self.get_column_id(expected_column_name)
|
|
399
400
|
if not expected_column_id:
|
|
400
|
-
raise ValueError(
|
|
401
|
+
raise ValueError(
|
|
402
|
+
f"Column '{expected_column_name}' not found in dataset"
|
|
403
|
+
)
|
|
401
404
|
mapping["expected_response"] = expected_column_id
|
|
402
405
|
|
|
403
406
|
eval_config = {
|
|
@@ -408,20 +411,16 @@ class DatasetClient(APIKeyAuth):
|
|
|
408
411
|
"config": {
|
|
409
412
|
"mapping": mapping,
|
|
410
413
|
"config": config or {},
|
|
411
|
-
"reasonColumn": reason_column
|
|
412
|
-
}
|
|
414
|
+
"reasonColumn": reason_column,
|
|
415
|
+
},
|
|
413
416
|
}
|
|
414
417
|
|
|
415
418
|
url = f"{self._base_url}/model-hub/develops/{str(self.dataset_config.id)}/add_user_eval/"
|
|
416
419
|
self.request(
|
|
417
|
-
config=RequestConfig(
|
|
418
|
-
|
|
419
|
-
url=url,
|
|
420
|
-
json=eval_config
|
|
421
|
-
),
|
|
422
|
-
response_handler=DatasetResponseHandler
|
|
420
|
+
config=RequestConfig(method=HttpMethod.POST, url=url, json=eval_config),
|
|
421
|
+
response_handler=DatasetResponseHandler,
|
|
423
422
|
)
|
|
424
|
-
|
|
423
|
+
|
|
425
424
|
return self
|
|
426
425
|
|
|
427
426
|
def add_optimization(
|
|
@@ -429,7 +428,7 @@ class DatasetClient(APIKeyAuth):
|
|
|
429
428
|
optimization_name: str,
|
|
430
429
|
prompt_column_name: str,
|
|
431
430
|
optimize_type: str = "PROMPT_TEMPLATE",
|
|
432
|
-
model_config: Optional[Dict[str, Any]] = None
|
|
431
|
+
model_config: Optional[Dict[str, Any]] = None,
|
|
433
432
|
) -> "DatasetClient":
|
|
434
433
|
|
|
435
434
|
if not self.dataset_config or not self.dataset_config.id:
|
|
@@ -437,17 +436,17 @@ class DatasetClient(APIKeyAuth):
|
|
|
437
436
|
|
|
438
437
|
column_id = self.get_column_id(prompt_column_name)
|
|
439
438
|
if not column_id:
|
|
440
|
-
raise ValueError(
|
|
439
|
+
raise ValueError(
|
|
440
|
+
f"Prompt Column '{prompt_column_name}' not found in dataset"
|
|
441
|
+
)
|
|
441
442
|
|
|
442
|
-
#requires mapping input for userEvalTemplateIds
|
|
443
|
+
# requires mapping input for userEvalTemplateIds
|
|
443
444
|
metrics_url = f"{self._base_url}/model-hub/metrics/by-column/"
|
|
444
445
|
metrics_response = self.request(
|
|
445
446
|
config=RequestConfig(
|
|
446
|
-
method=HttpMethod.GET,
|
|
447
|
-
url=metrics_url,
|
|
448
|
-
params={"column_id": column_id}
|
|
447
|
+
method=HttpMethod.GET, url=metrics_url, params={"column_id": column_id}
|
|
449
448
|
),
|
|
450
|
-
response_handler=DatasetResponseHandler
|
|
449
|
+
response_handler=DatasetResponseHandler,
|
|
451
450
|
)
|
|
452
451
|
|
|
453
452
|
eval_template_ids = []
|
|
@@ -456,7 +455,9 @@ class DatasetClient(APIKeyAuth):
|
|
|
456
455
|
eval_template_ids.append(metric["id"])
|
|
457
456
|
|
|
458
457
|
if not eval_template_ids:
|
|
459
|
-
raise ValueError(
|
|
458
|
+
raise ValueError(
|
|
459
|
+
f"No evaluation templates found for column '{prompt_column_name}'"
|
|
460
|
+
)
|
|
460
461
|
|
|
461
462
|
payload = {
|
|
462
463
|
"name": optimization_name,
|
|
@@ -464,19 +465,15 @@ class DatasetClient(APIKeyAuth):
|
|
|
464
465
|
"optimizeType": optimize_type,
|
|
465
466
|
"userEvalTemplateIds": eval_template_ids,
|
|
466
467
|
"datasetId": str(self.dataset_config.id),
|
|
467
|
-
"modelConfig": model_config or {}
|
|
468
|
+
"modelConfig": model_config or {},
|
|
468
469
|
}
|
|
469
470
|
|
|
470
471
|
opt_url = f"{self._base_url}/model-hub/optimisation/create/"
|
|
471
472
|
self.request(
|
|
472
|
-
config=RequestConfig(
|
|
473
|
-
|
|
474
|
-
url=opt_url,
|
|
475
|
-
json=payload
|
|
476
|
-
),
|
|
477
|
-
response_handler=DatasetResponseHandler
|
|
473
|
+
config=RequestConfig(method=HttpMethod.POST, url=opt_url, json=payload),
|
|
474
|
+
response_handler=DatasetResponseHandler,
|
|
478
475
|
)
|
|
479
|
-
|
|
476
|
+
|
|
480
477
|
return self
|
|
481
478
|
|
|
482
479
|
# Protected internal methods
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
from fi.evals.evaluator import EvalClient # noqa: F401
|
|
2
|
+
from fi.evals.protect import ProtectClient # noqa: F401
|
|
2
3
|
from fi.evals.templates import (
|
|
3
4
|
PII,
|
|
4
5
|
AgentJudge,
|
|
@@ -130,4 +131,5 @@ __all__ = [
|
|
|
130
131
|
"Toxicity",
|
|
131
132
|
"TranslationAccuracy",
|
|
132
133
|
"EvalClient",
|
|
134
|
+
"ProtectClient",
|
|
133
135
|
]
|