futureagi 0.4.3__tar.gz → 0.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of futureagi might be problematic. Click here for more details.
- {futureagi-0.4.3 → futureagi-0.5.0}/PKG-INFO +24 -2
- {futureagi-0.4.3 → futureagi-0.5.0}/fi/datasets/client.py +11 -7
- {futureagi-0.4.3 → futureagi-0.5.0}/fi/evals/protect.py +20 -14
- futureagi-0.5.0/fi/integrations/otel/__init__.py +50 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/anthropic/__init__.py +117 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/anthropic/_stream.py +556 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/anthropic/_utils.py +108 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/anthropic/_with_span.py +119 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/anthropic/_wrappers.py +856 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/anthropic/examples/anthropic.sh +21 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/anthropic/examples/end_to_end_tool_use.py +125 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/anthropic/examples/multiple_tool_calling.py +65 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/anthropic/examples/new-1.py +238 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/anthropic/examples/requirements.txt +1 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/anthropic/examples/sync_completions.py +32 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/anthropic/examples/sync_messages.py +29 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/anthropic/examples/test-script.py +61 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/anthropic/version.py +1 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/__generated__/__init__.py +0 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/__generated__/anthropic/__init__.py +0 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/__generated__/anthropic/_types.py +176 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/__init__.py +491 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/_proxy.py +303 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/_wrappers.py +90 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/examples/bedrock.sh +17 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/examples/bedrock_converse.py +65 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/examples/bedrock_multimodal.py +78 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/examples/new-1.py +234 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/examples/new-2.py +134 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/package.py +2 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/py.typed +0 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/utils/__init__.py +93 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/utils/anthropic/__init__.py +46 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/utils/anthropic/_messages.py +439 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/bedrock/version.py +1 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/crewai/__init__.py +87 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/crewai/_wrappers.py +364 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/crewai/examples/crew_ai.py +154 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/crewai/version.py +1 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/dspy/__init__.py +739 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/dspy/examples/dspy.sh +19 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/dspy/examples/dspy_basic_qa.py +34 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/dspy/examples/lm.py +18 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/dspy/examples/rag_module.py +44 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/dspy/examples/requirements.txt +1 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/dspy/package.py +2 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/dspy/py.typed +0 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/dspy/version.py +1 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/groq/__init__.py +68 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/groq/_request_attributes_extractor.py +138 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/groq/_response_attributes_extractor.py +109 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/groq/_utils.py +95 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/groq/_with_span.py +94 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/groq/_wrappers.py +266 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/groq/examples/async_chat_completions.py +66 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/groq/examples/chat_completions.py +58 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/groq/examples/chat_completions_with_tool.py +136 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/groq/examples/groq.sh +19 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/groq/version.py +1 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/haystack/__init__.py +68 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/haystack/_wrappers.py +805 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/haystack/examples/building_fallbacks_with_conditional_routing.py +131 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/haystack/examples/cohere_reranker.py +46 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/haystack/examples/filtering_documents_with_metadata.py +71 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/haystack/examples/haystack.sh +23 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/haystack/examples/haystack_rag_pipeline.py +79 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/haystack/examples/qa_rag_pipeline.py +88 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/haystack/examples/requirements.txt +4 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/haystack/examples/tool_calling.py +81 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/haystack/examples/web_questions.py +51 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/haystack/version.py +1 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/instructor/__init__.py +61 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/instructor/_wrappers.py +249 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/instructor/examples/test_instructor.py +40 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/instructor/version.py +1 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/langchain/__init__.py +148 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/langchain/_tracer.py +1088 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/langchain/examples/chain_metadata.py +45 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/langchain/examples/chat_prompt_template.py +21 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/langchain/examples/custom_retriever.py +59 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/langchain/examples/exchange_rate_tool.py +50 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/langchain/examples/langchain.sh +23 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/langchain/examples/langgraph_agent_supervisor.py +156 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/langchain/examples/multimodal.py +76 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/langchain/examples/openai_chat_stream.py +40 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/langchain/examples/pathway_retriever.py +23 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/langchain/examples/requirements.txt +6 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/langchain/package.py +2 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/langchain/version.py +1 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/litellm/__init__.py +386 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/litellm/examples/litellm_test.py +78 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/litellm/package.py +2 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/litellm/py.typed +0 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/litellm/version.py +1 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/llama_index/__init__.py +129 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/llama_index/_callback.py +795 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/llama_index/_handler.py +1073 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/llama_index/examples/agent_calculator_tools.py +37 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/llama_index/examples/chat_engine_stream.py +38 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/llama_index/examples/chroma_query_engine.py +70 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/llama_index/examples/ingestion_pipeline.py +47 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/llama_index/examples/llama_index.sh +24 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/llama_index/examples/multimodal_stream_chat.py +62 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/llama_index/examples/openai_stream_chat.py +25 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/llama_index/examples/requirements.txt +15 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/llama_index/examples/text_to_sql.py +217 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/llama_index/examples/workflow.py +49 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/llama_index/package.py +2 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/llama_index/py.typed +0 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/llama_index/version.py +1 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/__init__.py +139 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/_chat_wrapper.py +416 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/_request_attributes_extractor.py +109 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/_response_accumulator.py +208 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/_response_attributes_extractor.py +127 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/_stream.py +183 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/_utils.py +78 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/_with_span.py +86 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/examples/agent_completions.py +26 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/examples/chat_completions.py +48 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/examples/chat_completions_async.py +57 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/examples/chat_completions_async_streaming.py +37 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/examples/chat_completions_streaming.py +33 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/examples/mistralai.sh +21 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/examples/requirements.txt +1 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/package.py +2 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/py.typed +0 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/mistralai/version.py +1 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/__init__.py +61 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/_request.py +530 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/_request_attributes_extractor.py +238 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/_response_accumulator.py +275 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/_response_attributes_extractor.py +259 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/_span_io_handler.py +156 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/_stream.py +204 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/_utils.py +135 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/_with_span.py +95 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/examples/chat_completions.py +45 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/examples/chat_completions_async_stream.py +57 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/examples/chat_completions_async_stream_with_raw_response.py +36 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/examples/chat_completions_multimodal.py +42 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/examples/chat_completions_stream.py +28 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/examples/chat_completions_stream_with_raw_response.py +28 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/examples/chat_completions_with_function_calling.py +64 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/examples/chat_completions_with_raw_response.py +24 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/examples/chat_completions_with_streaming_response.py +28 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/examples/embeddings.py +22 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/examples/openai.sh +27 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/examples/openai_tool.py +49 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/examples/requirements.txt +1 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/package.py +2 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/py.typed +0 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/test-requirements.txt +6 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/openai/version.py +1 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/__init__.py +56 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/_accumulator.py +113 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/_instrumentation_status.py +7 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/_proxy.py +303 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/_wrapper.py +644 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/examples/audio_transcription.py +30 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/examples/audio_understanding.py +31 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/examples/basic_generation.py +24 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/examples/chat.py +22 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/examples/document_understanding.py +30 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/examples/function_calling.py +57 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/examples/image_from_cloud_storage.py +28 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/examples/requirements.txt +1 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/examples/system_instructions.py +27 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/examples/vertexai.model.count_tokens.py +20 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/examples/vertexai.model.generate_content.py +26 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/examples/vertexai.sh +28 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/examples/video_and_audio_understanding.py +30 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/package.py +2 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/py.typed +0 -0
- futureagi-0.5.0/fi/integrations/otel/frameworks/vertexai/version.py +1 -0
- futureagi-0.5.0/fi/integrations/otel/instrumentation/.gitignore +14 -0
- futureagi-0.5.0/fi/integrations/otel/instrumentation/__init__.py +26 -0
- futureagi-0.5.0/fi/integrations/otel/instrumentation/config.py +429 -0
- futureagi-0.5.0/fi/integrations/otel/instrumentation/context_attributes.py +292 -0
- futureagi-0.5.0/fi/integrations/otel/instrumentation/helpers.py +23 -0
- futureagi-0.5.0/fi/integrations/otel/instrumentation/logging.py +22 -0
- futureagi-0.5.0/fi/integrations/otel/instrumentation/version.py +1 -0
- futureagi-0.5.0/fi/integrations/otel/otel.py +485 -0
- futureagi-0.5.0/fi/integrations/otel/settings.py +121 -0
- futureagi-0.5.0/fi/integrations/otel/types.py +762 -0
- futureagi-0.5.0/fi/test/eval_tags.py +59 -0
- futureagi-0.5.0/fi/test/test-3.py +208 -0
- {futureagi-0.4.3 → futureagi-0.5.0}/fi/utils/constants.py +4 -0
- futureagi-0.5.0/pyproject.toml +58 -0
- futureagi-0.4.3/pyproject.toml +0 -37
- {futureagi-0.4.3 → futureagi-0.5.0}/LICENSE.md +0 -0
- {futureagi-0.4.3 → futureagi-0.5.0}/README.md +0 -0
- {futureagi-0.4.3 → futureagi-0.5.0}/fi/__init__.py +0 -0
- {futureagi-0.4.3 → futureagi-0.5.0}/fi/api/__init__.py +0 -0
- {futureagi-0.4.3 → futureagi-0.5.0}/fi/api/apikeys.py +0 -0
- {futureagi-0.4.3 → futureagi-0.5.0}/fi/api/auth.py +0 -0
- {futureagi-0.4.3 → futureagi-0.5.0}/fi/api/types.py +0 -0
- {futureagi-0.4.3 → futureagi-0.5.0}/fi/client.py +0 -0
- {futureagi-0.4.3 → futureagi-0.5.0}/fi/datasets/__init__.py +0 -0
- {futureagi-0.4.3 → futureagi-0.5.0}/fi/datasets/types.py +0 -0
- {futureagi-0.4.3 → futureagi-0.5.0}/fi/evals/__init__.py +0 -0
- {futureagi-0.4.3 → futureagi-0.5.0}/fi/evals/evaluator.py +0 -0
- {futureagi-0.4.3 → futureagi-0.5.0}/fi/evals/templates.py +0 -0
- {futureagi-0.4.3 → futureagi-0.5.0}/fi/evals/types.py +0 -0
- {futureagi-0.4.3 → futureagi-0.5.0}/fi/integrations/__init__.py +0 -0
- {futureagi-0.4.3 → futureagi-0.5.0}/fi/integrations/providers/__init__.py +0 -0
- {futureagi-0.4.3 → futureagi-0.5.0}/fi/integrations/providers/types.py +0 -0
- {futureagi-0.4.3 → futureagi-0.5.0}/fi/prompt/__init__.py +0 -0
- {futureagi-0.4.3 → futureagi-0.5.0}/fi/prompt/client.py +0 -0
- {futureagi-0.4.3 → futureagi-0.5.0}/fi/prompt/types.py +0 -0
- {futureagi-0.4.3 → futureagi-0.5.0}/fi/testcases/__init__.py +0 -0
- {futureagi-0.4.3 → futureagi-0.5.0}/fi/testcases/conversational_test_case.py +0 -0
- {futureagi-0.4.3 → futureagi-0.5.0}/fi/testcases/general.py +0 -0
- {futureagi-0.4.3 → futureagi-0.5.0}/fi/testcases/llm_test_case.py +0 -0
- {futureagi-0.4.3 → futureagi-0.5.0}/fi/testcases/mllm_test_case.py +0 -0
- {futureagi-0.4.3 → futureagi-0.5.0}/fi/testcases/utils.py +0 -0
- {futureagi-0.4.3 → futureagi-0.5.0}/fi/utils/__init__.py +0 -0
- {futureagi-0.4.3 → futureagi-0.5.0}/fi/utils/errors.py +0 -0
- {futureagi-0.4.3 → futureagi-0.5.0}/fi/utils/executor.py +0 -0
- {futureagi-0.4.3 → futureagi-0.5.0}/fi/utils/logging.py +0 -0
- {futureagi-0.4.3 → futureagi-0.5.0}/fi/utils/routes.py +0 -0
- {futureagi-0.4.3 → futureagi-0.5.0}/fi/utils/types.py +0 -0
- {futureagi-0.4.3 → futureagi-0.5.0}/fi/utils/utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
2
|
Name: futureagi
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.5.0
|
|
4
4
|
Summary: We help GenAI teams maintain high-accuracy for their Models in production.
|
|
5
5
|
Author: Future AGI
|
|
6
6
|
Author-email: no-reply@futureagi.com
|
|
@@ -10,10 +10,31 @@ Classifier: Programming Language :: Python :: 3.10
|
|
|
10
10
|
Classifier: Programming Language :: Python :: 3.11
|
|
11
11
|
Classifier: Programming Language :: Python :: 3.12
|
|
12
12
|
Classifier: Programming Language :: Python :: 3.13
|
|
13
|
+
Requires-Dist: anthropic (>=0.42.0,<0.43.0)
|
|
14
|
+
Requires-Dist: boto3 (>=1.28.57)
|
|
15
|
+
Requires-Dist: botocore (>=1.34.116) ; python_version > "3.10" and python_version < "3.13"
|
|
13
16
|
Requires-Dist: certifi (==2024.6.2)
|
|
14
17
|
Requires-Dist: charset-normalizer (==3.3.2)
|
|
18
|
+
Requires-Dist: crewai (>0.19.0) ; python_version >= "3.10" and python_version < "3.13"
|
|
19
|
+
Requires-Dist: dacite (>=1.9.1,<2.0.0)
|
|
20
|
+
Requires-Dist: dspy (>=2.5.0)
|
|
21
|
+
Requires-Dist: google-cloud-aiplatform (>=1.49.0) ; python_version > "3.9" and python_version < "4.0"
|
|
22
|
+
Requires-Dist: groq (>=0.9.0)
|
|
23
|
+
Requires-Dist: haystack-ai (>=2.0.0)
|
|
15
24
|
Requires-Dist: idna (==3.7)
|
|
25
|
+
Requires-Dist: instructor (>=0.0.1) ; python_version > "3.9" and python_version < "4.0"
|
|
26
|
+
Requires-Dist: langchain-community[all] (>=0.3.9,<0.4.0) ; python_version >= "3.9" and python_version < "4.0"
|
|
27
|
+
Requires-Dist: langchain[all] (>=0.3.9,<0.4.0) ; python_version >= "3.9" and python_version < "4.0"
|
|
28
|
+
Requires-Dist: litellm (>=1.43.0)
|
|
29
|
+
Requires-Dist: llama-index-core (>=0.10.43) ; python_version > "3.9" and python_version < "4.0"
|
|
30
|
+
Requires-Dist: mistralai (>=1.2.6,<2.0.0) ; python_version > "3.9" and python_version < "4.0"
|
|
16
31
|
Requires-Dist: numpy (==1.26.4)
|
|
32
|
+
Requires-Dist: openai (>=1.57.1,<2.0.0)
|
|
33
|
+
Requires-Dist: openinference-instrumentation-openai (>=0.1.18,<0.2.0) ; python_version >= "3.9" and python_version < "3.13"
|
|
34
|
+
Requires-Dist: opentelemetry-api (>=1.29.0,<2.0.0)
|
|
35
|
+
Requires-Dist: opentelemetry-exporter-otlp-proto-http (>=1.29.0,<2.0.0)
|
|
36
|
+
Requires-Dist: opentelemetry-instrumentation-openai (>=0.34.1,<0.35.0) ; python_version >= "3.9" and python_version < "4"
|
|
37
|
+
Requires-Dist: opentelemetry-sdk (>=1.29.0,<2.0.0)
|
|
17
38
|
Requires-Dist: pandas (==2.2.2)
|
|
18
39
|
Requires-Dist: pydantic (>=2.9.2,<3.0.0)
|
|
19
40
|
Requires-Dist: python-dateutil (==2.9.0.post0)
|
|
@@ -21,6 +42,7 @@ Requires-Dist: pytz (==2024.1)
|
|
|
21
42
|
Requires-Dist: requests (==2.32.3)
|
|
22
43
|
Requires-Dist: requests-futures (==1.0.1)
|
|
23
44
|
Requires-Dist: six (==1.16.0)
|
|
45
|
+
Requires-Dist: tiktoken (>=0.8.0,<0.9.0)
|
|
24
46
|
Requires-Dist: tqdm (>=4.67.1,<5.0.0)
|
|
25
47
|
Requires-Dist: tzdata (==2024.1)
|
|
26
48
|
Requires-Dist: urllib3 (==2.2.2)
|
|
@@ -29,7 +29,6 @@ from fi.utils.utils import get_tempfile_path
|
|
|
29
29
|
|
|
30
30
|
|
|
31
31
|
class DatasetResponseHandler(ResponseHandler[DatasetConfig, DatasetTable]):
|
|
32
|
-
|
|
33
32
|
"""Handles responses for dataset requests"""
|
|
34
33
|
|
|
35
34
|
@classmethod
|
|
@@ -60,9 +59,11 @@ class DatasetResponseHandler(ResponseHandler[DatasetConfig, DatasetTable]):
|
|
|
60
59
|
data_type=column["dataType"],
|
|
61
60
|
source=column["originType"],
|
|
62
61
|
source_id=column["sourceId"],
|
|
63
|
-
is_frozen=
|
|
64
|
-
|
|
65
|
-
|
|
62
|
+
is_frozen=(
|
|
63
|
+
column["isFrozen"]["isFrozen"]
|
|
64
|
+
if column["isFrozen"] is not None
|
|
65
|
+
else False
|
|
66
|
+
),
|
|
66
67
|
is_visible=column["isVisible"],
|
|
67
68
|
eval_tags=column["evalTag"],
|
|
68
69
|
average_score=column["averageScore"],
|
|
@@ -81,9 +82,11 @@ class DatasetResponseHandler(ResponseHandler[DatasetConfig, DatasetTable]):
|
|
|
81
82
|
column_id=column_id,
|
|
82
83
|
row_id=row_id,
|
|
83
84
|
value=value.get("cellValue"),
|
|
84
|
-
value_infos=
|
|
85
|
-
|
|
86
|
-
|
|
85
|
+
value_infos=(
|
|
86
|
+
[value.get("valueInfos")]
|
|
87
|
+
if value.get("valueInfos")
|
|
88
|
+
else None
|
|
89
|
+
),
|
|
87
90
|
metadata=value.get("metadata"),
|
|
88
91
|
status=value.get("status"),
|
|
89
92
|
failure_reason=value.get("failureReason"),
|
|
@@ -119,6 +122,7 @@ class DatasetResponseHandler(ResponseHandler[DatasetConfig, DatasetTable]):
|
|
|
119
122
|
response.raise_for_status()
|
|
120
123
|
if response.status_code == 403:
|
|
121
124
|
raise InvalidAuthError()
|
|
125
|
+
raise Exception(response.json())
|
|
122
126
|
|
|
123
127
|
|
|
124
128
|
class DatasetClient(APIKeyAuth):
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import time
|
|
2
2
|
from concurrent.futures import ThreadPoolExecutor, TimeoutError, as_completed
|
|
3
3
|
from typing import Dict, List, Optional, Tuple
|
|
4
|
+
import copy
|
|
4
5
|
|
|
5
6
|
from fi.evals import EvalClient
|
|
6
7
|
from fi.evals.templates import (
|
|
@@ -178,7 +179,7 @@ class ProtectClient:
|
|
|
178
179
|
|
|
179
180
|
Args:
|
|
180
181
|
inputs: Single string or list of strings to evaluate
|
|
181
|
-
|
|
182
|
+
protect_rules_copy: List of protection rule dictionaries. Each rule must contain:
|
|
182
183
|
- metric: str, name of the metric to evaluate (e.g., 'Toxicity', 'Tone', 'Sexism')
|
|
183
184
|
- contains: List[str], values to check for in the evaluation results
|
|
184
185
|
- type: str, either 'any' or 'all'
|
|
@@ -190,9 +191,14 @@ class ProtectClient:
|
|
|
190
191
|
List of protection messages for failed rules or ["All checks passed"] if no rules are triggered
|
|
191
192
|
|
|
192
193
|
Raises:
|
|
193
|
-
ValueError: If inputs or
|
|
194
|
+
ValueError: If inputs or protect_rules_copy don't match the required structure
|
|
194
195
|
TypeError: If inputs contains non-string objects
|
|
195
196
|
"""
|
|
197
|
+
timeout = timeout * 1000
|
|
198
|
+
|
|
199
|
+
protect_rules_copy = copy.deepcopy(protect_rules)
|
|
200
|
+
|
|
201
|
+
|
|
196
202
|
# Validate inputs
|
|
197
203
|
if inputs is None:
|
|
198
204
|
raise ValueError("inputs cannot be None")
|
|
@@ -219,17 +225,17 @@ class ProtectClient:
|
|
|
219
225
|
# Convert inputs to TestCase instances
|
|
220
226
|
test_cases = [TestCase(input=input_text) for input_text in inputs]
|
|
221
227
|
|
|
222
|
-
# Validate
|
|
223
|
-
if not isinstance(
|
|
224
|
-
raise TypeError("
|
|
228
|
+
# Validate protect_rules_copy
|
|
229
|
+
if not isinstance(protect_rules_copy, list):
|
|
230
|
+
raise TypeError("protect_rules_copy must be a list")
|
|
225
231
|
|
|
226
|
-
if not
|
|
227
|
-
raise ValueError("
|
|
232
|
+
if not protect_rules_copy:
|
|
233
|
+
raise ValueError("protect_rules_copy cannot be empty")
|
|
228
234
|
|
|
229
235
|
valid_metrics = set(self.metric_map.keys())
|
|
230
236
|
valid_types = {"any", "all"}
|
|
231
237
|
|
|
232
|
-
for i, rule in enumerate(
|
|
238
|
+
for i, rule in enumerate(protect_rules_copy):
|
|
233
239
|
|
|
234
240
|
if not isinstance(rule, dict):
|
|
235
241
|
raise TypeError(f"Rule at index {i} must be a dictionary")
|
|
@@ -333,8 +339,8 @@ class ProtectClient:
|
|
|
333
339
|
|
|
334
340
|
# results = []
|
|
335
341
|
BATCH_SIZE = 5 # Maximum number of concurrent rule checks
|
|
336
|
-
if len(
|
|
337
|
-
BATCH_SIZE = len(
|
|
342
|
+
if len(protect_rules_copy) < BATCH_SIZE:
|
|
343
|
+
BATCH_SIZE = len(protect_rules_copy)
|
|
338
344
|
|
|
339
345
|
total_timeout = timeout / 1000 # in seconds
|
|
340
346
|
start_time = time.time()
|
|
@@ -346,18 +352,18 @@ class ProtectClient:
|
|
|
346
352
|
# try:
|
|
347
353
|
bool_check_fail = False
|
|
348
354
|
for test_case in test_cases:
|
|
349
|
-
for i in range(0, len(
|
|
355
|
+
for i in range(0, len(protect_rules_copy), BATCH_SIZE):
|
|
350
356
|
# Calculate remaining time
|
|
351
357
|
elapsed_time = time.time() - start_time
|
|
352
358
|
remaining_time = max(0, total_timeout - elapsed_time)
|
|
353
359
|
|
|
354
360
|
if remaining_time <= 0:
|
|
355
361
|
# Add remaining rules to uncompleted list
|
|
356
|
-
remaining_rules = [rule["metric"] for rule in
|
|
362
|
+
remaining_rules = [rule["metric"] for rule in protect_rules_copy[i:]]
|
|
357
363
|
all_uncompleted_rules.extend(remaining_rules)
|
|
358
364
|
break
|
|
359
365
|
|
|
360
|
-
rules_batch =
|
|
366
|
+
rules_batch = protect_rules_copy[i : i + BATCH_SIZE]
|
|
361
367
|
(
|
|
362
368
|
messages,
|
|
363
369
|
completed,
|
|
@@ -388,7 +394,7 @@ class ProtectClient:
|
|
|
388
394
|
"time_taken": elapsed_time,
|
|
389
395
|
}
|
|
390
396
|
|
|
391
|
-
if len(ans["uncompleted_rules"]) == len(
|
|
397
|
+
if len(ans["uncompleted_rules"]) == len(protect_rules_copy):
|
|
392
398
|
ans["reason"] = "No checks completed"
|
|
393
399
|
|
|
394
400
|
if bool_check_fail:
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
from opentelemetry.sdk.resources import Resource
|
|
2
|
+
|
|
3
|
+
from .frameworks.anthropic import AnthropicInstrumentor
|
|
4
|
+
from .frameworks.bedrock import BedrockInstrumentor
|
|
5
|
+
from .frameworks.crewai import CrewAIInstrumentor
|
|
6
|
+
from .frameworks.dspy import DSPyInstrumentor
|
|
7
|
+
from .frameworks.groq import GroqInstrumentor
|
|
8
|
+
from .frameworks.haystack import HaystackInstrumentor
|
|
9
|
+
from .frameworks.instructor import InstructorInstrumentor
|
|
10
|
+
from .frameworks.langchain import LangChainInstrumentor
|
|
11
|
+
from .frameworks.litellm import LiteLLMInstrumentor
|
|
12
|
+
from .frameworks.llama_index import LlamaIndexInstrumentor
|
|
13
|
+
from .frameworks.mistralai import MistralAIInstrumentor
|
|
14
|
+
from .frameworks.openai import OpenAIInstrumentor
|
|
15
|
+
from .frameworks.vertexai import VertexAIInstrumentor
|
|
16
|
+
from .otel import (
|
|
17
|
+
PROJECT_NAME,
|
|
18
|
+
PROJECT_TYPE,
|
|
19
|
+
PROJECT_VERSION_NAME,
|
|
20
|
+
BatchSpanProcessor,
|
|
21
|
+
HTTPSpanExporter,
|
|
22
|
+
SimpleSpanProcessor,
|
|
23
|
+
TracerProvider,
|
|
24
|
+
register,
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
__all__ = [
|
|
28
|
+
"TracerProvider",
|
|
29
|
+
"SimpleSpanProcessor",
|
|
30
|
+
"BatchSpanProcessor",
|
|
31
|
+
"HTTPSpanExporter",
|
|
32
|
+
"Resource",
|
|
33
|
+
"PROJECT_NAME",
|
|
34
|
+
"PROJECT_TYPE",
|
|
35
|
+
"PROJECT_VERSION_NAME",
|
|
36
|
+
"register",
|
|
37
|
+
"LangChainInstrumentor",
|
|
38
|
+
"OpenAIInstrumentor",
|
|
39
|
+
"AnthropicInstrumentor",
|
|
40
|
+
"BedrockInstrumentor",
|
|
41
|
+
"CrewAIInstrumentor",
|
|
42
|
+
"DSPyInstrumentor",
|
|
43
|
+
"GroqInstrumentor",
|
|
44
|
+
"HaystackInstrumentor",
|
|
45
|
+
"InstructorInstrumentor",
|
|
46
|
+
"LiteLLMInstrumentor",
|
|
47
|
+
"LlamaIndexInstrumentor",
|
|
48
|
+
"MistralAIInstrumentor",
|
|
49
|
+
"VertexAIInstrumentor",
|
|
50
|
+
]
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from typing import Any, Collection
|
|
3
|
+
|
|
4
|
+
from opentelemetry import trace as trace_api
|
|
5
|
+
from opentelemetry.instrumentation.instrumentor import ( # type: ignore[attr-defined]
|
|
6
|
+
BaseInstrumentor,
|
|
7
|
+
)
|
|
8
|
+
from wrapt import wrap_function_wrapper
|
|
9
|
+
|
|
10
|
+
from fi.integrations.otel.frameworks.anthropic._wrappers import (
|
|
11
|
+
_AsyncCompletionsWrapper,
|
|
12
|
+
_AsyncMessagesWrapper,
|
|
13
|
+
_CompletionsWrapper,
|
|
14
|
+
_MessagesCountTokensWrapper,
|
|
15
|
+
_MessagesWrapper,
|
|
16
|
+
)
|
|
17
|
+
from fi.integrations.otel.frameworks.anthropic.version import __version__
|
|
18
|
+
from fi.integrations.otel.instrumentation import FITracer, TraceConfig
|
|
19
|
+
|
|
20
|
+
logger = logging.getLogger(__name__)
|
|
21
|
+
|
|
22
|
+
_instruments = ("anthropic >= 0.30.0",)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class AnthropicInstrumentor(BaseInstrumentor): # type: ignore[misc]
|
|
26
|
+
"""An instrumentor for the Anthropic framework."""
|
|
27
|
+
|
|
28
|
+
__slots__ = (
|
|
29
|
+
"_original_completions_create",
|
|
30
|
+
"_original_async_completions_create",
|
|
31
|
+
"_original_messages_create",
|
|
32
|
+
"_original_async_messages_create",
|
|
33
|
+
"_instruments",
|
|
34
|
+
"_tracer",
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
def instrumentation_dependencies(self) -> Collection[str]:
|
|
38
|
+
return _instruments
|
|
39
|
+
|
|
40
|
+
def _instrument(self, **kwargs: Any) -> None:
|
|
41
|
+
print("Starting instrumentation process...")
|
|
42
|
+
from anthropic.resources.completions import AsyncCompletions, Completions
|
|
43
|
+
from anthropic.resources.messages import AsyncMessages, Messages
|
|
44
|
+
|
|
45
|
+
print(f"Initializing with tracer_provider: {kwargs.get('tracer_provider')}")
|
|
46
|
+
if not (tracer_provider := kwargs.get("tracer_provider")):
|
|
47
|
+
tracer_provider = trace_api.get_tracer_provider()
|
|
48
|
+
if not (config := kwargs.get("config")):
|
|
49
|
+
config = TraceConfig()
|
|
50
|
+
else:
|
|
51
|
+
assert isinstance(config, TraceConfig)
|
|
52
|
+
|
|
53
|
+
print("Creating FITracer instance...")
|
|
54
|
+
self._tracer = FITracer(
|
|
55
|
+
trace_api.get_tracer(__name__, __version__, tracer_provider),
|
|
56
|
+
config=config,
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
print("Wrapping Completions.create...")
|
|
60
|
+
self._original_completions_create = Completions.create
|
|
61
|
+
wrap_function_wrapper(
|
|
62
|
+
module="anthropic.resources.completions",
|
|
63
|
+
name="Completions.create",
|
|
64
|
+
wrapper=_CompletionsWrapper(tracer=self._tracer),
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
print("Wrapping AsyncCompletions.create...")
|
|
68
|
+
self._original_async_completions_create = AsyncCompletions.create
|
|
69
|
+
wrap_function_wrapper(
|
|
70
|
+
module="anthropic.resources.completions",
|
|
71
|
+
name="AsyncCompletions.create",
|
|
72
|
+
wrapper=_AsyncCompletionsWrapper(tracer=self._tracer),
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
print("Wrapping Messages.create...")
|
|
76
|
+
self._original_messages_create = Messages.create
|
|
77
|
+
wrap_function_wrapper(
|
|
78
|
+
module="anthropic.resources.messages",
|
|
79
|
+
name="Messages.create",
|
|
80
|
+
wrapper=_MessagesWrapper(tracer=self._tracer),
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
print("Wrapping AsyncMessages.create...")
|
|
84
|
+
self._original_async_messages_create = AsyncMessages.create
|
|
85
|
+
wrap_function_wrapper(
|
|
86
|
+
module="anthropic.resources.messages",
|
|
87
|
+
name="AsyncMessages.create",
|
|
88
|
+
wrapper=_AsyncMessagesWrapper(tracer=self._tracer),
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
print("Wrapping Messages.count_tokens...")
|
|
92
|
+
wrap_function_wrapper(
|
|
93
|
+
module="anthropic.resources.messages",
|
|
94
|
+
name="Messages.count_tokens",
|
|
95
|
+
wrapper=_MessagesCountTokensWrapper(tracer=self._tracer),
|
|
96
|
+
)
|
|
97
|
+
print("Instrumentation complete!")
|
|
98
|
+
|
|
99
|
+
def _uninstrument(self, **kwargs: Any) -> None:
|
|
100
|
+
print("Starting uninstrumentation process...")
|
|
101
|
+
from anthropic.resources.completions import AsyncCompletions, Completions
|
|
102
|
+
from anthropic.resources.messages import AsyncMessages, Messages
|
|
103
|
+
|
|
104
|
+
if self._original_completions_create is not None:
|
|
105
|
+
print("Restoring original Completions.create...")
|
|
106
|
+
Completions.create = self._original_completions_create # type: ignore[method-assign]
|
|
107
|
+
if self._original_async_completions_create is not None:
|
|
108
|
+
print("Restoring original AsyncCompletions.create...")
|
|
109
|
+
AsyncCompletions.create = self._original_async_completions_create # type: ignore[method-assign]
|
|
110
|
+
|
|
111
|
+
if self._original_messages_create is not None:
|
|
112
|
+
print("Restoring original Messages.create...")
|
|
113
|
+
Messages.create = self._original_messages_create # type: ignore[method-assign]
|
|
114
|
+
if self._original_async_messages_create is not None:
|
|
115
|
+
print("Restoring original AsyncMessages.create...")
|
|
116
|
+
AsyncMessages.create = self._original_async_messages_create # type: ignore[method-assign]
|
|
117
|
+
print("Uninstrumentation complete!")
|