opik 1.8.39__py3-none-any.whl → 1.9.71__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- opik/__init__.py +19 -3
- opik/anonymizer/__init__.py +5 -0
- opik/anonymizer/anonymizer.py +12 -0
- opik/anonymizer/factory.py +80 -0
- opik/anonymizer/recursive_anonymizer.py +64 -0
- opik/anonymizer/rules.py +56 -0
- opik/anonymizer/rules_anonymizer.py +35 -0
- opik/api_objects/attachment/attachment_context.py +36 -0
- opik/api_objects/attachment/attachments_extractor.py +153 -0
- opik/api_objects/attachment/client.py +1 -0
- opik/api_objects/attachment/converters.py +2 -0
- opik/api_objects/attachment/decoder.py +18 -0
- opik/api_objects/attachment/decoder_base64.py +83 -0
- opik/api_objects/attachment/decoder_helpers.py +137 -0
- opik/api_objects/data_helpers.py +79 -0
- opik/api_objects/dataset/dataset.py +64 -4
- opik/api_objects/dataset/rest_operations.py +11 -2
- opik/api_objects/experiment/experiment.py +57 -57
- opik/api_objects/experiment/experiment_item.py +2 -1
- opik/api_objects/experiment/experiments_client.py +64 -0
- opik/api_objects/experiment/helpers.py +35 -11
- opik/api_objects/experiment/rest_operations.py +65 -5
- opik/api_objects/helpers.py +8 -5
- opik/api_objects/local_recording.py +81 -0
- opik/api_objects/opik_client.py +600 -108
- opik/api_objects/opik_query_language.py +39 -5
- opik/api_objects/prompt/__init__.py +12 -2
- opik/api_objects/prompt/base_prompt.py +69 -0
- opik/api_objects/prompt/base_prompt_template.py +29 -0
- opik/api_objects/prompt/chat/__init__.py +1 -0
- opik/api_objects/prompt/chat/chat_prompt.py +210 -0
- opik/api_objects/prompt/chat/chat_prompt_template.py +350 -0
- opik/api_objects/prompt/chat/content_renderer_registry.py +203 -0
- opik/api_objects/prompt/client.py +189 -47
- opik/api_objects/prompt/text/__init__.py +1 -0
- opik/api_objects/prompt/text/prompt.py +174 -0
- opik/api_objects/prompt/{prompt_template.py → text/prompt_template.py} +10 -6
- opik/api_objects/prompt/types.py +23 -0
- opik/api_objects/search_helpers.py +89 -0
- opik/api_objects/span/span_data.py +35 -25
- opik/api_objects/threads/threads_client.py +39 -5
- opik/api_objects/trace/trace_client.py +52 -2
- opik/api_objects/trace/trace_data.py +15 -24
- opik/api_objects/validation_helpers.py +3 -3
- opik/cli/__init__.py +5 -0
- opik/cli/__main__.py +6 -0
- opik/cli/configure.py +66 -0
- opik/cli/exports/__init__.py +131 -0
- opik/cli/exports/dataset.py +278 -0
- opik/cli/exports/experiment.py +784 -0
- opik/cli/exports/project.py +685 -0
- opik/cli/exports/prompt.py +578 -0
- opik/cli/exports/utils.py +406 -0
- opik/cli/harbor.py +39 -0
- opik/cli/healthcheck.py +21 -0
- opik/cli/imports/__init__.py +439 -0
- opik/cli/imports/dataset.py +143 -0
- opik/cli/imports/experiment.py +1192 -0
- opik/cli/imports/project.py +262 -0
- opik/cli/imports/prompt.py +177 -0
- opik/cli/imports/utils.py +280 -0
- opik/cli/main.py +49 -0
- opik/cli/proxy.py +93 -0
- opik/cli/usage_report/__init__.py +16 -0
- opik/cli/usage_report/charts.py +783 -0
- opik/cli/usage_report/cli.py +274 -0
- opik/cli/usage_report/constants.py +9 -0
- opik/cli/usage_report/extraction.py +749 -0
- opik/cli/usage_report/pdf.py +244 -0
- opik/cli/usage_report/statistics.py +78 -0
- opik/cli/usage_report/utils.py +235 -0
- opik/config.py +13 -7
- opik/configurator/configure.py +17 -0
- opik/datetime_helpers.py +12 -0
- opik/decorator/arguments_helpers.py +9 -1
- opik/decorator/base_track_decorator.py +205 -133
- opik/decorator/context_manager/span_context_manager.py +123 -0
- opik/decorator/context_manager/trace_context_manager.py +84 -0
- opik/decorator/opik_args/__init__.py +13 -0
- opik/decorator/opik_args/api_classes.py +71 -0
- opik/decorator/opik_args/helpers.py +120 -0
- opik/decorator/span_creation_handler.py +25 -6
- opik/dict_utils.py +3 -3
- opik/evaluation/__init__.py +13 -2
- opik/evaluation/engine/engine.py +272 -75
- opik/evaluation/engine/evaluation_tasks_executor.py +6 -3
- opik/evaluation/engine/helpers.py +31 -6
- opik/evaluation/engine/metrics_evaluator.py +237 -0
- opik/evaluation/evaluation_result.py +168 -2
- opik/evaluation/evaluator.py +533 -62
- opik/evaluation/metrics/__init__.py +103 -4
- opik/evaluation/metrics/aggregated_metric.py +35 -6
- opik/evaluation/metrics/base_metric.py +1 -1
- opik/evaluation/metrics/conversation/__init__.py +48 -0
- opik/evaluation/metrics/conversation/conversation_thread_metric.py +56 -2
- opik/evaluation/metrics/conversation/g_eval_wrappers.py +19 -0
- opik/evaluation/metrics/conversation/helpers.py +14 -15
- opik/evaluation/metrics/conversation/heuristics/__init__.py +14 -0
- opik/evaluation/metrics/conversation/heuristics/degeneration/__init__.py +3 -0
- opik/evaluation/metrics/conversation/heuristics/degeneration/metric.py +189 -0
- opik/evaluation/metrics/conversation/heuristics/degeneration/phrases.py +12 -0
- opik/evaluation/metrics/conversation/heuristics/knowledge_retention/__init__.py +3 -0
- opik/evaluation/metrics/conversation/heuristics/knowledge_retention/metric.py +172 -0
- opik/evaluation/metrics/conversation/llm_judges/__init__.py +32 -0
- opik/evaluation/metrics/conversation/{conversational_coherence → llm_judges/conversational_coherence}/metric.py +22 -17
- opik/evaluation/metrics/conversation/{conversational_coherence → llm_judges/conversational_coherence}/templates.py +1 -1
- opik/evaluation/metrics/conversation/llm_judges/g_eval_wrappers.py +442 -0
- opik/evaluation/metrics/conversation/{session_completeness → llm_judges/session_completeness}/metric.py +13 -7
- opik/evaluation/metrics/conversation/{session_completeness → llm_judges/session_completeness}/templates.py +1 -1
- opik/evaluation/metrics/conversation/llm_judges/user_frustration/__init__.py +0 -0
- opik/evaluation/metrics/conversation/{user_frustration → llm_judges/user_frustration}/metric.py +21 -14
- opik/evaluation/metrics/conversation/{user_frustration → llm_judges/user_frustration}/templates.py +1 -1
- opik/evaluation/metrics/conversation/types.py +4 -5
- opik/evaluation/metrics/conversation_types.py +9 -0
- opik/evaluation/metrics/heuristics/bertscore.py +107 -0
- opik/evaluation/metrics/heuristics/bleu.py +35 -15
- opik/evaluation/metrics/heuristics/chrf.py +127 -0
- opik/evaluation/metrics/heuristics/contains.py +47 -11
- opik/evaluation/metrics/heuristics/distribution_metrics.py +331 -0
- opik/evaluation/metrics/heuristics/gleu.py +113 -0
- opik/evaluation/metrics/heuristics/language_adherence.py +123 -0
- opik/evaluation/metrics/heuristics/meteor.py +119 -0
- opik/evaluation/metrics/heuristics/prompt_injection.py +150 -0
- opik/evaluation/metrics/heuristics/readability.py +129 -0
- opik/evaluation/metrics/heuristics/rouge.py +26 -9
- opik/evaluation/metrics/heuristics/spearman.py +88 -0
- opik/evaluation/metrics/heuristics/tone.py +155 -0
- opik/evaluation/metrics/heuristics/vader_sentiment.py +77 -0
- opik/evaluation/metrics/llm_judges/answer_relevance/metric.py +20 -5
- opik/evaluation/metrics/llm_judges/context_precision/metric.py +20 -6
- opik/evaluation/metrics/llm_judges/context_recall/metric.py +20 -6
- opik/evaluation/metrics/llm_judges/g_eval/__init__.py +5 -0
- opik/evaluation/metrics/llm_judges/g_eval/metric.py +219 -68
- opik/evaluation/metrics/llm_judges/g_eval/parser.py +102 -52
- opik/evaluation/metrics/llm_judges/g_eval/presets.py +209 -0
- opik/evaluation/metrics/llm_judges/g_eval_presets/__init__.py +36 -0
- opik/evaluation/metrics/llm_judges/g_eval_presets/agent_assessment.py +77 -0
- opik/evaluation/metrics/llm_judges/g_eval_presets/bias_classifier.py +181 -0
- opik/evaluation/metrics/llm_judges/g_eval_presets/compliance_risk.py +41 -0
- opik/evaluation/metrics/llm_judges/g_eval_presets/prompt_uncertainty.py +41 -0
- opik/evaluation/metrics/llm_judges/g_eval_presets/qa_suite.py +146 -0
- opik/evaluation/metrics/llm_judges/hallucination/metric.py +16 -3
- opik/evaluation/metrics/llm_judges/llm_juries/__init__.py +3 -0
- opik/evaluation/metrics/llm_judges/llm_juries/metric.py +76 -0
- opik/evaluation/metrics/llm_judges/moderation/metric.py +16 -4
- opik/evaluation/metrics/llm_judges/structure_output_compliance/__init__.py +0 -0
- opik/evaluation/metrics/llm_judges/structure_output_compliance/metric.py +144 -0
- opik/evaluation/metrics/llm_judges/structure_output_compliance/parser.py +79 -0
- opik/evaluation/metrics/llm_judges/structure_output_compliance/schema.py +15 -0
- opik/evaluation/metrics/llm_judges/structure_output_compliance/template.py +50 -0
- opik/evaluation/metrics/llm_judges/syc_eval/__init__.py +0 -0
- opik/evaluation/metrics/llm_judges/syc_eval/metric.py +252 -0
- opik/evaluation/metrics/llm_judges/syc_eval/parser.py +82 -0
- opik/evaluation/metrics/llm_judges/syc_eval/template.py +155 -0
- opik/evaluation/metrics/llm_judges/trajectory_accuracy/metric.py +20 -5
- opik/evaluation/metrics/llm_judges/usefulness/metric.py +16 -4
- opik/evaluation/metrics/ragas_metric.py +43 -23
- opik/evaluation/models/__init__.py +8 -0
- opik/evaluation/models/base_model.py +107 -1
- opik/evaluation/models/langchain/langchain_chat_model.py +15 -7
- opik/evaluation/models/langchain/message_converters.py +97 -15
- opik/evaluation/models/litellm/litellm_chat_model.py +156 -29
- opik/evaluation/models/litellm/util.py +125 -0
- opik/evaluation/models/litellm/warning_filters.py +16 -4
- opik/evaluation/models/model_capabilities.py +187 -0
- opik/evaluation/models/models_factory.py +25 -3
- opik/evaluation/preprocessing.py +92 -0
- opik/evaluation/report.py +70 -12
- opik/evaluation/rest_operations.py +49 -45
- opik/evaluation/samplers/__init__.py +4 -0
- opik/evaluation/samplers/base_dataset_sampler.py +40 -0
- opik/evaluation/samplers/random_dataset_sampler.py +48 -0
- opik/evaluation/score_statistics.py +66 -0
- opik/evaluation/scorers/__init__.py +4 -0
- opik/evaluation/scorers/scorer_function.py +55 -0
- opik/evaluation/scorers/scorer_wrapper_metric.py +130 -0
- opik/evaluation/test_case.py +3 -2
- opik/evaluation/test_result.py +1 -0
- opik/evaluation/threads/evaluator.py +31 -3
- opik/evaluation/threads/helpers.py +3 -2
- opik/evaluation/types.py +9 -1
- opik/exceptions.py +33 -0
- opik/file_upload/file_uploader.py +13 -0
- opik/file_upload/upload_options.py +2 -0
- opik/hooks/__init__.py +23 -0
- opik/hooks/anonymizer_hook.py +36 -0
- opik/hooks/httpx_client_hook.py +112 -0
- opik/httpx_client.py +12 -9
- opik/id_helpers.py +18 -0
- opik/integrations/adk/graph/subgraph_edges_builders.py +1 -2
- opik/integrations/adk/helpers.py +16 -7
- opik/integrations/adk/legacy_opik_tracer.py +7 -4
- opik/integrations/adk/opik_tracer.py +14 -1
- opik/integrations/adk/patchers/adk_otel_tracer/opik_adk_otel_tracer.py +7 -3
- opik/integrations/adk/recursive_callback_injector.py +4 -7
- opik/integrations/bedrock/converse/__init__.py +0 -0
- opik/integrations/bedrock/converse/chunks_aggregator.py +188 -0
- opik/integrations/bedrock/{converse_decorator.py → converse/converse_decorator.py} +4 -3
- opik/integrations/bedrock/invoke_agent_decorator.py +5 -4
- opik/integrations/bedrock/invoke_model/__init__.py +0 -0
- opik/integrations/bedrock/invoke_model/chunks_aggregator/__init__.py +78 -0
- opik/integrations/bedrock/invoke_model/chunks_aggregator/api.py +45 -0
- opik/integrations/bedrock/invoke_model/chunks_aggregator/base.py +23 -0
- opik/integrations/bedrock/invoke_model/chunks_aggregator/claude.py +121 -0
- opik/integrations/bedrock/invoke_model/chunks_aggregator/format_detector.py +107 -0
- opik/integrations/bedrock/invoke_model/chunks_aggregator/llama.py +108 -0
- opik/integrations/bedrock/invoke_model/chunks_aggregator/mistral.py +118 -0
- opik/integrations/bedrock/invoke_model/chunks_aggregator/nova.py +99 -0
- opik/integrations/bedrock/invoke_model/invoke_model_decorator.py +178 -0
- opik/integrations/bedrock/invoke_model/response_types.py +34 -0
- opik/integrations/bedrock/invoke_model/stream_wrappers.py +122 -0
- opik/integrations/bedrock/invoke_model/usage_converters.py +87 -0
- opik/integrations/bedrock/invoke_model/usage_extraction.py +108 -0
- opik/integrations/bedrock/opik_tracker.py +42 -4
- opik/integrations/bedrock/types.py +19 -0
- opik/integrations/crewai/crewai_decorator.py +8 -51
- opik/integrations/crewai/opik_tracker.py +31 -10
- opik/integrations/crewai/patchers/__init__.py +5 -0
- opik/integrations/crewai/patchers/flow.py +118 -0
- opik/integrations/crewai/patchers/litellm_completion.py +30 -0
- opik/integrations/crewai/patchers/llm_client.py +207 -0
- opik/integrations/dspy/callback.py +80 -17
- opik/integrations/dspy/parsers.py +168 -0
- opik/integrations/harbor/__init__.py +17 -0
- opik/integrations/harbor/experiment_service.py +269 -0
- opik/integrations/harbor/opik_tracker.py +528 -0
- opik/integrations/haystack/opik_connector.py +2 -2
- opik/integrations/haystack/opik_tracer.py +3 -7
- opik/integrations/langchain/__init__.py +3 -1
- opik/integrations/langchain/helpers.py +96 -0
- opik/integrations/langchain/langgraph_async_context_bridge.py +131 -0
- opik/integrations/langchain/langgraph_tracer_injector.py +88 -0
- opik/integrations/langchain/opik_encoder_extension.py +1 -1
- opik/integrations/langchain/opik_tracer.py +474 -229
- opik/integrations/litellm/__init__.py +5 -0
- opik/integrations/litellm/completion_chunks_aggregator.py +115 -0
- opik/integrations/litellm/litellm_completion_decorator.py +242 -0
- opik/integrations/litellm/opik_tracker.py +43 -0
- opik/integrations/litellm/stream_patchers.py +151 -0
- opik/integrations/llama_index/callback.py +146 -107
- opik/integrations/openai/agents/opik_tracing_processor.py +1 -2
- opik/integrations/openai/openai_chat_completions_decorator.py +2 -16
- opik/integrations/openai/opik_tracker.py +1 -1
- opik/integrations/sagemaker/auth.py +5 -1
- opik/llm_usage/google_usage.py +3 -1
- opik/llm_usage/opik_usage.py +7 -8
- opik/llm_usage/opik_usage_factory.py +4 -2
- opik/logging_messages.py +6 -0
- opik/message_processing/batching/base_batcher.py +14 -21
- opik/message_processing/batching/batch_manager.py +22 -10
- opik/message_processing/batching/batch_manager_constuctors.py +10 -0
- opik/message_processing/batching/batchers.py +59 -27
- opik/message_processing/batching/flushing_thread.py +0 -3
- opik/message_processing/emulation/__init__.py +0 -0
- opik/message_processing/emulation/emulator_message_processor.py +578 -0
- opik/message_processing/emulation/local_emulator_message_processor.py +140 -0
- opik/message_processing/emulation/models.py +162 -0
- opik/message_processing/encoder_helpers.py +79 -0
- opik/message_processing/messages.py +56 -1
- opik/message_processing/preprocessing/__init__.py +0 -0
- opik/message_processing/preprocessing/attachments_preprocessor.py +70 -0
- opik/message_processing/preprocessing/batching_preprocessor.py +53 -0
- opik/message_processing/preprocessing/constants.py +1 -0
- opik/message_processing/preprocessing/file_upload_preprocessor.py +38 -0
- opik/message_processing/preprocessing/preprocessor.py +36 -0
- opik/message_processing/processors/__init__.py +0 -0
- opik/message_processing/processors/attachments_extraction_processor.py +146 -0
- opik/message_processing/processors/message_processors.py +92 -0
- opik/message_processing/processors/message_processors_chain.py +96 -0
- opik/message_processing/{message_processors.py → processors/online_message_processor.py} +85 -29
- opik/message_processing/queue_consumer.py +9 -3
- opik/message_processing/streamer.py +71 -33
- opik/message_processing/streamer_constructors.py +43 -10
- opik/opik_context.py +16 -4
- opik/plugins/pytest/hooks.py +5 -3
- opik/rest_api/__init__.py +346 -15
- opik/rest_api/alerts/__init__.py +7 -0
- opik/rest_api/alerts/client.py +667 -0
- opik/rest_api/alerts/raw_client.py +1015 -0
- opik/rest_api/alerts/types/__init__.py +7 -0
- opik/rest_api/alerts/types/get_webhook_examples_request_alert_type.py +5 -0
- opik/rest_api/annotation_queues/__init__.py +4 -0
- opik/rest_api/annotation_queues/client.py +668 -0
- opik/rest_api/annotation_queues/raw_client.py +1019 -0
- opik/rest_api/automation_rule_evaluators/client.py +34 -2
- opik/rest_api/automation_rule_evaluators/raw_client.py +24 -0
- opik/rest_api/client.py +15 -0
- opik/rest_api/dashboards/__init__.py +4 -0
- opik/rest_api/dashboards/client.py +462 -0
- opik/rest_api/dashboards/raw_client.py +648 -0
- opik/rest_api/datasets/client.py +1310 -44
- opik/rest_api/datasets/raw_client.py +2269 -358
- opik/rest_api/experiments/__init__.py +2 -2
- opik/rest_api/experiments/client.py +191 -5
- opik/rest_api/experiments/raw_client.py +301 -7
- opik/rest_api/experiments/types/__init__.py +4 -1
- opik/rest_api/experiments/types/experiment_update_status.py +5 -0
- opik/rest_api/experiments/types/experiment_update_type.py +5 -0
- opik/rest_api/experiments/types/experiment_write_status.py +5 -0
- opik/rest_api/feedback_definitions/types/find_feedback_definitions_request_type.py +1 -1
- opik/rest_api/llm_provider_key/client.py +20 -0
- opik/rest_api/llm_provider_key/raw_client.py +20 -0
- opik/rest_api/llm_provider_key/types/provider_api_key_write_provider.py +1 -1
- opik/rest_api/manual_evaluation/__init__.py +4 -0
- opik/rest_api/manual_evaluation/client.py +347 -0
- opik/rest_api/manual_evaluation/raw_client.py +543 -0
- opik/rest_api/optimizations/client.py +145 -9
- opik/rest_api/optimizations/raw_client.py +237 -13
- opik/rest_api/optimizations/types/optimization_update_status.py +3 -1
- opik/rest_api/prompts/__init__.py +2 -2
- opik/rest_api/prompts/client.py +227 -6
- opik/rest_api/prompts/raw_client.py +331 -2
- opik/rest_api/prompts/types/__init__.py +3 -1
- opik/rest_api/prompts/types/create_prompt_version_detail_template_structure.py +5 -0
- opik/rest_api/prompts/types/prompt_write_template_structure.py +5 -0
- opik/rest_api/spans/__init__.py +0 -2
- opik/rest_api/spans/client.py +238 -76
- opik/rest_api/spans/raw_client.py +307 -95
- opik/rest_api/spans/types/__init__.py +0 -2
- opik/rest_api/traces/client.py +572 -161
- opik/rest_api/traces/raw_client.py +736 -229
- opik/rest_api/types/__init__.py +352 -17
- opik/rest_api/types/aggregation_data.py +1 -0
- opik/rest_api/types/alert.py +33 -0
- opik/rest_api/types/alert_alert_type.py +5 -0
- opik/rest_api/types/alert_page_public.py +24 -0
- opik/rest_api/types/alert_public.py +33 -0
- opik/rest_api/types/alert_public_alert_type.py +5 -0
- opik/rest_api/types/alert_trigger.py +27 -0
- opik/rest_api/types/alert_trigger_config.py +28 -0
- opik/rest_api/types/alert_trigger_config_public.py +28 -0
- opik/rest_api/types/alert_trigger_config_public_type.py +10 -0
- opik/rest_api/types/alert_trigger_config_type.py +10 -0
- opik/rest_api/types/alert_trigger_config_write.py +22 -0
- opik/rest_api/types/alert_trigger_config_write_type.py +10 -0
- opik/rest_api/types/alert_trigger_event_type.py +19 -0
- opik/rest_api/types/alert_trigger_public.py +27 -0
- opik/rest_api/types/alert_trigger_public_event_type.py +19 -0
- opik/rest_api/types/alert_trigger_write.py +23 -0
- opik/rest_api/types/alert_trigger_write_event_type.py +19 -0
- opik/rest_api/types/alert_write.py +28 -0
- opik/rest_api/types/alert_write_alert_type.py +5 -0
- opik/rest_api/types/annotation_queue.py +42 -0
- opik/rest_api/types/annotation_queue_batch.py +27 -0
- opik/rest_api/types/annotation_queue_item_ids.py +19 -0
- opik/rest_api/types/annotation_queue_page_public.py +28 -0
- opik/rest_api/types/annotation_queue_public.py +38 -0
- opik/rest_api/types/annotation_queue_public_scope.py +5 -0
- opik/rest_api/types/annotation_queue_reviewer.py +20 -0
- opik/rest_api/types/annotation_queue_reviewer_public.py +20 -0
- opik/rest_api/types/annotation_queue_scope.py +5 -0
- opik/rest_api/types/annotation_queue_write.py +31 -0
- opik/rest_api/types/annotation_queue_write_scope.py +5 -0
- opik/rest_api/types/audio_url.py +19 -0
- opik/rest_api/types/audio_url_public.py +19 -0
- opik/rest_api/types/audio_url_write.py +19 -0
- opik/rest_api/types/automation_rule_evaluator.py +62 -2
- opik/rest_api/types/automation_rule_evaluator_llm_as_judge.py +2 -0
- opik/rest_api/types/automation_rule_evaluator_llm_as_judge_public.py +2 -0
- opik/rest_api/types/automation_rule_evaluator_llm_as_judge_write.py +2 -0
- opik/rest_api/types/automation_rule_evaluator_object_object_public.py +155 -0
- opik/rest_api/types/automation_rule_evaluator_page_public.py +3 -2
- opik/rest_api/types/automation_rule_evaluator_public.py +57 -2
- opik/rest_api/types/automation_rule_evaluator_span_llm_as_judge.py +22 -0
- opik/rest_api/types/automation_rule_evaluator_span_llm_as_judge_public.py +22 -0
- opik/rest_api/types/automation_rule_evaluator_span_llm_as_judge_write.py +22 -0
- opik/rest_api/types/automation_rule_evaluator_span_user_defined_metric_python.py +22 -0
- opik/rest_api/types/automation_rule_evaluator_span_user_defined_metric_python_public.py +22 -0
- opik/rest_api/types/automation_rule_evaluator_span_user_defined_metric_python_write.py +22 -0
- opik/rest_api/types/automation_rule_evaluator_trace_thread_llm_as_judge.py +2 -0
- opik/rest_api/types/automation_rule_evaluator_trace_thread_llm_as_judge_public.py +2 -0
- opik/rest_api/types/automation_rule_evaluator_trace_thread_llm_as_judge_write.py +2 -0
- opik/rest_api/types/automation_rule_evaluator_trace_thread_user_defined_metric_python.py +2 -0
- opik/rest_api/types/automation_rule_evaluator_trace_thread_user_defined_metric_python_public.py +2 -0
- opik/rest_api/types/automation_rule_evaluator_trace_thread_user_defined_metric_python_write.py +2 -0
- opik/rest_api/types/automation_rule_evaluator_update.py +51 -1
- opik/rest_api/types/automation_rule_evaluator_update_llm_as_judge.py +2 -0
- opik/rest_api/types/automation_rule_evaluator_update_span_llm_as_judge.py +22 -0
- opik/rest_api/types/automation_rule_evaluator_update_span_user_defined_metric_python.py +22 -0
- opik/rest_api/types/automation_rule_evaluator_update_trace_thread_llm_as_judge.py +2 -0
- opik/rest_api/types/automation_rule_evaluator_update_trace_thread_user_defined_metric_python.py +2 -0
- opik/rest_api/types/automation_rule_evaluator_update_user_defined_metric_python.py +2 -0
- opik/rest_api/types/automation_rule_evaluator_user_defined_metric_python.py +2 -0
- opik/rest_api/types/automation_rule_evaluator_user_defined_metric_python_public.py +2 -0
- opik/rest_api/types/automation_rule_evaluator_user_defined_metric_python_write.py +2 -0
- opik/rest_api/types/automation_rule_evaluator_write.py +51 -1
- opik/rest_api/types/boolean_feedback_definition.py +25 -0
- opik/rest_api/types/boolean_feedback_definition_create.py +20 -0
- opik/rest_api/types/boolean_feedback_definition_public.py +25 -0
- opik/rest_api/types/boolean_feedback_definition_update.py +20 -0
- opik/rest_api/types/boolean_feedback_detail.py +29 -0
- opik/rest_api/types/boolean_feedback_detail_create.py +29 -0
- opik/rest_api/types/boolean_feedback_detail_public.py +29 -0
- opik/rest_api/types/boolean_feedback_detail_update.py +29 -0
- opik/rest_api/types/dashboard_page_public.py +24 -0
- opik/rest_api/types/dashboard_public.py +30 -0
- opik/rest_api/types/dataset.py +4 -0
- opik/rest_api/types/dataset_expansion.py +42 -0
- opik/rest_api/types/dataset_expansion_response.py +39 -0
- opik/rest_api/types/dataset_item.py +2 -0
- opik/rest_api/types/dataset_item_changes_public.py +5 -0
- opik/rest_api/types/dataset_item_compare.py +2 -0
- opik/rest_api/types/dataset_item_filter.py +27 -0
- opik/rest_api/types/dataset_item_filter_operator.py +21 -0
- opik/rest_api/types/dataset_item_page_compare.py +5 -0
- opik/rest_api/types/dataset_item_page_public.py +5 -0
- opik/rest_api/types/dataset_item_public.py +2 -0
- opik/rest_api/types/dataset_item_update.py +39 -0
- opik/rest_api/types/dataset_item_write.py +1 -0
- opik/rest_api/types/dataset_public.py +4 -0
- opik/rest_api/types/dataset_public_status.py +5 -0
- opik/rest_api/types/dataset_status.py +5 -0
- opik/rest_api/types/dataset_version_diff.py +22 -0
- opik/rest_api/types/dataset_version_diff_stats.py +24 -0
- opik/rest_api/types/dataset_version_page_public.py +23 -0
- opik/rest_api/types/dataset_version_public.py +59 -0
- opik/rest_api/types/dataset_version_summary.py +46 -0
- opik/rest_api/types/dataset_version_summary_public.py +46 -0
- opik/rest_api/types/experiment.py +7 -2
- opik/rest_api/types/experiment_group_response.py +2 -0
- opik/rest_api/types/experiment_public.py +7 -2
- opik/rest_api/types/experiment_public_status.py +5 -0
- opik/rest_api/types/experiment_score.py +20 -0
- opik/rest_api/types/experiment_score_public.py +20 -0
- opik/rest_api/types/experiment_score_write.py +20 -0
- opik/rest_api/types/experiment_status.py +5 -0
- opik/rest_api/types/feedback.py +25 -1
- opik/rest_api/types/feedback_create.py +20 -1
- opik/rest_api/types/feedback_object_public.py +27 -1
- opik/rest_api/types/feedback_public.py +25 -1
- opik/rest_api/types/feedback_score_batch_item.py +2 -1
- opik/rest_api/types/feedback_score_batch_item_thread.py +2 -1
- opik/rest_api/types/feedback_score_public.py +4 -0
- opik/rest_api/types/feedback_update.py +20 -1
- opik/rest_api/types/group_content_with_aggregations.py +1 -0
- opik/rest_api/types/group_detail.py +19 -0
- opik/rest_api/types/group_details.py +20 -0
- opik/rest_api/types/guardrail.py +1 -0
- opik/rest_api/types/guardrail_write.py +1 -0
- opik/rest_api/types/ids_holder.py +19 -0
- opik/rest_api/types/image_url.py +20 -0
- opik/rest_api/types/image_url_public.py +20 -0
- opik/rest_api/types/image_url_write.py +20 -0
- opik/rest_api/types/llm_as_judge_message.py +5 -1
- opik/rest_api/types/llm_as_judge_message_content.py +26 -0
- opik/rest_api/types/llm_as_judge_message_content_public.py +26 -0
- opik/rest_api/types/llm_as_judge_message_content_write.py +26 -0
- opik/rest_api/types/llm_as_judge_message_public.py +5 -1
- opik/rest_api/types/llm_as_judge_message_write.py +5 -1
- opik/rest_api/types/llm_as_judge_model_parameters.py +3 -0
- opik/rest_api/types/llm_as_judge_model_parameters_public.py +3 -0
- opik/rest_api/types/llm_as_judge_model_parameters_write.py +3 -0
- opik/rest_api/types/manual_evaluation_request.py +38 -0
- opik/rest_api/types/manual_evaluation_request_entity_type.py +5 -0
- opik/rest_api/types/manual_evaluation_response.py +27 -0
- opik/rest_api/types/optimization.py +4 -2
- opik/rest_api/types/optimization_public.py +4 -2
- opik/rest_api/types/optimization_public_status.py +3 -1
- opik/rest_api/types/optimization_status.py +3 -1
- opik/rest_api/types/optimization_studio_config.py +27 -0
- opik/rest_api/types/optimization_studio_config_public.py +27 -0
- opik/rest_api/types/optimization_studio_config_write.py +27 -0
- opik/rest_api/types/optimization_studio_log.py +22 -0
- opik/rest_api/types/optimization_write.py +4 -2
- opik/rest_api/types/optimization_write_status.py +3 -1
- opik/rest_api/types/project.py +1 -0
- opik/rest_api/types/project_detailed.py +1 -0
- opik/rest_api/types/project_reference.py +31 -0
- opik/rest_api/types/project_reference_public.py +31 -0
- opik/rest_api/types/project_stats_summary_item.py +1 -0
- opik/rest_api/types/prompt.py +6 -0
- opik/rest_api/types/prompt_detail.py +6 -0
- opik/rest_api/types/prompt_detail_template_structure.py +5 -0
- opik/rest_api/types/prompt_public.py +6 -0
- opik/rest_api/types/prompt_public_template_structure.py +5 -0
- opik/rest_api/types/prompt_template_structure.py +5 -0
- opik/rest_api/types/prompt_version.py +3 -0
- opik/rest_api/types/prompt_version_detail.py +3 -0
- opik/rest_api/types/prompt_version_detail_template_structure.py +5 -0
- opik/rest_api/types/prompt_version_link.py +1 -0
- opik/rest_api/types/prompt_version_link_public.py +1 -0
- opik/rest_api/types/prompt_version_page_public.py +5 -0
- opik/rest_api/types/prompt_version_public.py +3 -0
- opik/rest_api/types/prompt_version_public_template_structure.py +5 -0
- opik/rest_api/types/prompt_version_template_structure.py +5 -0
- opik/rest_api/types/prompt_version_update.py +33 -0
- opik/rest_api/types/provider_api_key.py +9 -0
- opik/rest_api/types/provider_api_key_provider.py +1 -1
- opik/rest_api/types/provider_api_key_public.py +9 -0
- opik/rest_api/types/provider_api_key_public_provider.py +1 -1
- opik/rest_api/types/score_name.py +1 -0
- opik/rest_api/types/service_toggles_config.py +18 -0
- opik/rest_api/types/span.py +1 -2
- opik/rest_api/types/span_enrichment_options.py +31 -0
- opik/rest_api/types/span_experiment_item_bulk_write_view.py +1 -2
- opik/rest_api/types/span_filter.py +23 -0
- opik/rest_api/types/span_filter_operator.py +21 -0
- opik/rest_api/types/span_filter_write.py +23 -0
- opik/rest_api/types/span_filter_write_operator.py +21 -0
- opik/rest_api/types/span_llm_as_judge_code.py +27 -0
- opik/rest_api/types/span_llm_as_judge_code_public.py +27 -0
- opik/rest_api/types/span_llm_as_judge_code_write.py +27 -0
- opik/rest_api/types/span_public.py +1 -2
- opik/rest_api/types/span_update.py +46 -0
- opik/rest_api/types/span_user_defined_metric_python_code.py +20 -0
- opik/rest_api/types/span_user_defined_metric_python_code_public.py +20 -0
- opik/rest_api/types/span_user_defined_metric_python_code_write.py +20 -0
- opik/rest_api/types/span_write.py +1 -2
- opik/rest_api/types/studio_evaluation.py +20 -0
- opik/rest_api/types/studio_evaluation_public.py +20 -0
- opik/rest_api/types/studio_evaluation_write.py +20 -0
- opik/rest_api/types/studio_llm_model.py +21 -0
- opik/rest_api/types/studio_llm_model_public.py +21 -0
- opik/rest_api/types/studio_llm_model_write.py +21 -0
- opik/rest_api/types/studio_message.py +20 -0
- opik/rest_api/types/studio_message_public.py +20 -0
- opik/rest_api/types/studio_message_write.py +20 -0
- opik/rest_api/types/studio_metric.py +21 -0
- opik/rest_api/types/studio_metric_public.py +21 -0
- opik/rest_api/types/studio_metric_write.py +21 -0
- opik/rest_api/types/studio_optimizer.py +21 -0
- opik/rest_api/types/studio_optimizer_public.py +21 -0
- opik/rest_api/types/studio_optimizer_write.py +21 -0
- opik/rest_api/types/studio_prompt.py +20 -0
- opik/rest_api/types/studio_prompt_public.py +20 -0
- opik/rest_api/types/studio_prompt_write.py +20 -0
- opik/rest_api/types/trace.py +11 -2
- opik/rest_api/types/trace_enrichment_options.py +32 -0
- opik/rest_api/types/trace_experiment_item_bulk_write_view.py +1 -2
- opik/rest_api/types/trace_filter.py +23 -0
- opik/rest_api/types/trace_filter_operator.py +21 -0
- opik/rest_api/types/trace_filter_write.py +23 -0
- opik/rest_api/types/trace_filter_write_operator.py +21 -0
- opik/rest_api/types/trace_public.py +11 -2
- opik/rest_api/types/trace_thread_filter_write.py +23 -0
- opik/rest_api/types/trace_thread_filter_write_operator.py +21 -0
- opik/rest_api/types/trace_thread_identifier.py +1 -0
- opik/rest_api/types/trace_update.py +39 -0
- opik/rest_api/types/trace_write.py +1 -2
- opik/rest_api/types/value_entry.py +2 -0
- opik/rest_api/types/value_entry_compare.py +2 -0
- opik/rest_api/types/value_entry_experiment_item_bulk_write_view.py +2 -0
- opik/rest_api/types/value_entry_public.py +2 -0
- opik/rest_api/types/video_url.py +19 -0
- opik/rest_api/types/video_url_public.py +19 -0
- opik/rest_api/types/video_url_write.py +19 -0
- opik/rest_api/types/webhook.py +28 -0
- opik/rest_api/types/webhook_examples.py +19 -0
- opik/rest_api/types/webhook_public.py +28 -0
- opik/rest_api/types/webhook_test_result.py +23 -0
- opik/rest_api/types/webhook_test_result_status.py +5 -0
- opik/rest_api/types/webhook_write.py +23 -0
- opik/rest_api/types/welcome_wizard_tracking.py +22 -0
- opik/rest_api/types/workspace_configuration.py +5 -0
- opik/rest_api/welcome_wizard/__init__.py +4 -0
- opik/rest_api/welcome_wizard/client.py +195 -0
- opik/rest_api/welcome_wizard/raw_client.py +208 -0
- opik/rest_api/workspaces/client.py +14 -2
- opik/rest_api/workspaces/raw_client.py +10 -0
- opik/s3_httpx_client.py +14 -1
- opik/simulation/__init__.py +6 -0
- opik/simulation/simulated_user.py +99 -0
- opik/simulation/simulator.py +108 -0
- opik/synchronization.py +5 -6
- opik/{decorator/tracing_runtime_config.py → tracing_runtime_config.py} +6 -7
- opik/types.py +36 -0
- opik/validation/chat_prompt_messages.py +241 -0
- opik/validation/feedback_score.py +3 -3
- opik/validation/validator.py +28 -0
- opik-1.9.71.dist-info/METADATA +370 -0
- opik-1.9.71.dist-info/RECORD +1110 -0
- opik/api_objects/prompt/prompt.py +0 -112
- opik/cli.py +0 -193
- opik/hooks.py +0 -13
- opik/integrations/bedrock/chunks_aggregator.py +0 -55
- opik/integrations/bedrock/helpers.py +0 -8
- opik/rest_api/types/automation_rule_evaluator_object_public.py +0 -100
- opik/rest_api/types/json_node_experiment_item_bulk_write_view.py +0 -5
- opik-1.8.39.dist-info/METADATA +0 -339
- opik-1.8.39.dist-info/RECORD +0 -790
- /opik/{evaluation/metrics/conversation/conversational_coherence → decorator/context_manager}/__init__.py +0 -0
- /opik/evaluation/metrics/conversation/{session_completeness → llm_judges/conversational_coherence}/__init__.py +0 -0
- /opik/evaluation/metrics/conversation/{conversational_coherence → llm_judges/conversational_coherence}/schema.py +0 -0
- /opik/evaluation/metrics/conversation/{user_frustration → llm_judges/session_completeness}/__init__.py +0 -0
- /opik/evaluation/metrics/conversation/{session_completeness → llm_judges/session_completeness}/schema.py +0 -0
- /opik/evaluation/metrics/conversation/{user_frustration → llm_judges/user_frustration}/schema.py +0 -0
- /opik/integrations/bedrock/{stream_wrappers.py → converse/stream_wrappers.py} +0 -0
- /opik/rest_api/{spans/types → types}/span_update_type.py +0 -0
- {opik-1.8.39.dist-info → opik-1.9.71.dist-info}/WHEEL +0 -0
- {opik-1.8.39.dist-info → opik-1.9.71.dist-info}/entry_points.txt +0 -0
- {opik-1.8.39.dist-info → opik-1.9.71.dist-info}/licenses/LICENSE +0 -0
- {opik-1.8.39.dist-info → opik-1.9.71.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,783 @@
|
|
|
1
|
+
"""Chart creation functions for usage report module."""
|
|
2
|
+
|
|
3
|
+
import datetime
|
|
4
|
+
import os
|
|
5
|
+
import time
|
|
6
|
+
import traceback
|
|
7
|
+
from typing import Any, Dict, List, Optional, Tuple
|
|
8
|
+
|
|
9
|
+
from rich.console import Console
|
|
10
|
+
|
|
11
|
+
from .utils import extract_metric_data
|
|
12
|
+
|
|
13
|
+
console = Console()
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _get_top_projects_and_others(
|
|
17
|
+
projects: List[Dict[str, Any]],
|
|
18
|
+
project_names: List[str],
|
|
19
|
+
metric_data: List[List[float]],
|
|
20
|
+
top_n: int = 12,
|
|
21
|
+
) -> Tuple[List[int], List[float], List[str], List]:
|
|
22
|
+
"""
|
|
23
|
+
Identify top N projects by total usage and group the rest as "Others".
|
|
24
|
+
|
|
25
|
+
Args:
|
|
26
|
+
projects: List of project dictionaries
|
|
27
|
+
project_names: List of project names
|
|
28
|
+
metric_data: List of lists, where each inner list contains values for one period
|
|
29
|
+
top_n: Number of top projects to show individually
|
|
30
|
+
|
|
31
|
+
Returns:
|
|
32
|
+
Tuple of (top_project_indices, others_data, labels, colors)
|
|
33
|
+
- top_project_indices: List of indices for top projects
|
|
34
|
+
- others_data: List of aggregated values for "Others" per period
|
|
35
|
+
- labels: List of labels (top project names + "Others")
|
|
36
|
+
- colors: List of colors for top projects + "Others"
|
|
37
|
+
"""
|
|
38
|
+
try:
|
|
39
|
+
import matplotlib.colors as mcolors
|
|
40
|
+
import matplotlib.pyplot as plt
|
|
41
|
+
except ImportError:
|
|
42
|
+
raise ImportError(
|
|
43
|
+
"matplotlib is required for chart generation. "
|
|
44
|
+
"Please install it with: pip install matplotlib"
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
# Calculate total usage per project across all periods
|
|
48
|
+
project_totals = []
|
|
49
|
+
for i in range(len(project_names)):
|
|
50
|
+
total = sum(metric_data[j][i] for j in range(len(metric_data)))
|
|
51
|
+
project_totals.append((i, total))
|
|
52
|
+
|
|
53
|
+
# Sort by total (descending) and get top N
|
|
54
|
+
project_totals.sort(key=lambda x: x[1], reverse=True)
|
|
55
|
+
top_indices = [idx for idx, _ in project_totals[:top_n]]
|
|
56
|
+
others_indices = [idx for idx, _ in project_totals[top_n:]]
|
|
57
|
+
|
|
58
|
+
# Create labels
|
|
59
|
+
labels = [project_names[i] for i in top_indices]
|
|
60
|
+
if others_indices:
|
|
61
|
+
labels.append(f"Others ({len(others_indices)} projects)")
|
|
62
|
+
|
|
63
|
+
# Aggregate "Others" data
|
|
64
|
+
others_data = []
|
|
65
|
+
if others_indices:
|
|
66
|
+
for period_idx in range(len(metric_data)):
|
|
67
|
+
others_total = sum(metric_data[period_idx][i] for i in others_indices)
|
|
68
|
+
others_data.append(others_total)
|
|
69
|
+
else:
|
|
70
|
+
others_data = [0.0] * len(metric_data)
|
|
71
|
+
|
|
72
|
+
# Generate colors for top projects + Others
|
|
73
|
+
colors_list = []
|
|
74
|
+
colormaps = [
|
|
75
|
+
plt.cm.tab20,
|
|
76
|
+
plt.cm.tab20b,
|
|
77
|
+
plt.cm.Set3,
|
|
78
|
+
plt.cm.Pastel1,
|
|
79
|
+
plt.cm.Pastel2,
|
|
80
|
+
plt.cm.Set1,
|
|
81
|
+
plt.cm.Set2,
|
|
82
|
+
]
|
|
83
|
+
|
|
84
|
+
for i in range(len(top_indices)):
|
|
85
|
+
if i < 20:
|
|
86
|
+
colors_list.append(colormaps[0](i))
|
|
87
|
+
elif i < 40:
|
|
88
|
+
colors_list.append(colormaps[1](i - 20))
|
|
89
|
+
elif i < 52:
|
|
90
|
+
colors_list.append(colormaps[2]((i - 40) % 12))
|
|
91
|
+
elif i < 61:
|
|
92
|
+
colors_list.append(colormaps[3]((i - 52) % 9))
|
|
93
|
+
elif i < 69:
|
|
94
|
+
colors_list.append(colormaps[4]((i - 61) % 8))
|
|
95
|
+
elif i < 78:
|
|
96
|
+
colors_list.append(colormaps[5]((i - 69) % 9))
|
|
97
|
+
elif i < 86:
|
|
98
|
+
colors_list.append(colormaps[6]((i - 78) % 8))
|
|
99
|
+
else:
|
|
100
|
+
hue = (i * 0.618033988749895) % 1.0
|
|
101
|
+
saturation = 0.6 + (i % 3) * 0.1
|
|
102
|
+
value = 0.85 + (i % 2) * 0.1
|
|
103
|
+
colors_list.append(mcolors.hsv_to_rgb([hue, saturation, value]))
|
|
104
|
+
|
|
105
|
+
# Add gray color for "Others"
|
|
106
|
+
if others_indices:
|
|
107
|
+
colors_list.append("#808080") # Gray for Others
|
|
108
|
+
|
|
109
|
+
return top_indices, others_data, labels, colors_list
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def create_charts(data: Dict[str, Any], output_dir: str = ".") -> None:
|
|
113
|
+
"""
|
|
114
|
+
Create stacked bar charts for trace count, token count, cost, experiment count, and dataset count.
|
|
115
|
+
|
|
116
|
+
Note: This function creates charts in memory but does not save them to disk.
|
|
117
|
+
Charts are generated and immediately closed. For saving charts, use create_individual_chart()
|
|
118
|
+
which is used by the PDF report generation.
|
|
119
|
+
|
|
120
|
+
Args:
|
|
121
|
+
data: The extracted data dictionary
|
|
122
|
+
output_dir: Directory parameter (kept for backward compatibility, not used)
|
|
123
|
+
"""
|
|
124
|
+
try:
|
|
125
|
+
import matplotlib.pyplot as plt
|
|
126
|
+
from matplotlib.ticker import FuncFormatter
|
|
127
|
+
except ImportError:
|
|
128
|
+
raise ImportError(
|
|
129
|
+
"matplotlib is required for chart generation. "
|
|
130
|
+
"Please install it with: pip install matplotlib"
|
|
131
|
+
)
|
|
132
|
+
|
|
133
|
+
# Get unit from data (default to month for backward compatibility)
|
|
134
|
+
unit = data.get("unit", "month")
|
|
135
|
+
|
|
136
|
+
# Prepare data for charts
|
|
137
|
+
projects = [
|
|
138
|
+
p for p in data["projects"] if "metrics_by_unit" in p and "error" not in p
|
|
139
|
+
]
|
|
140
|
+
if not projects:
|
|
141
|
+
console.print("[yellow]No project data available for charting.[/yellow]")
|
|
142
|
+
return
|
|
143
|
+
|
|
144
|
+
# Collect all time periods across all projects
|
|
145
|
+
all_periods_set = set()
|
|
146
|
+
for project in projects:
|
|
147
|
+
all_periods_set.update(project["metrics_by_unit"].keys())
|
|
148
|
+
all_periods: List[str] = sorted(all_periods_set)
|
|
149
|
+
|
|
150
|
+
if not all_periods:
|
|
151
|
+
console.print(f"[yellow]No {unit}ly data available for charting.[/yellow]")
|
|
152
|
+
return
|
|
153
|
+
|
|
154
|
+
# Prepare data arrays for each metric
|
|
155
|
+
project_names = [p["project_name"] for p in projects]
|
|
156
|
+
n_periods = len(all_periods)
|
|
157
|
+
|
|
158
|
+
# Helper function for token count aggregation
|
|
159
|
+
def aggregate_token_count(token_count: Any) -> float:
|
|
160
|
+
"""Aggregate token count: use total_tokens if available, otherwise sum all values."""
|
|
161
|
+
if isinstance(token_count, dict):
|
|
162
|
+
if "total_tokens" in token_count:
|
|
163
|
+
return float(token_count["total_tokens"])
|
|
164
|
+
else:
|
|
165
|
+
return (
|
|
166
|
+
sum(float(v) for v in token_count.values()) if token_count else 0.0
|
|
167
|
+
)
|
|
168
|
+
else:
|
|
169
|
+
return float(token_count) if token_count else 0.0
|
|
170
|
+
|
|
171
|
+
# Extract metric data using helper function
|
|
172
|
+
trace_data = extract_metric_data(projects, all_periods, "trace_count")
|
|
173
|
+
token_data = extract_metric_data(
|
|
174
|
+
projects, all_periods, "token_count", aggregate_token_count
|
|
175
|
+
)
|
|
176
|
+
cost_data = extract_metric_data(projects, all_periods, "cost")
|
|
177
|
+
span_data = extract_metric_data(projects, all_periods, "span_count")
|
|
178
|
+
|
|
179
|
+
# Format period labels for display based on unit
|
|
180
|
+
period_labels = []
|
|
181
|
+
for period in all_periods:
|
|
182
|
+
if unit == "month":
|
|
183
|
+
period_labels.append(
|
|
184
|
+
datetime.datetime.strptime(period, "%Y-%m").strftime("%b %Y")
|
|
185
|
+
)
|
|
186
|
+
elif unit == "week":
|
|
187
|
+
# Parse ISO week format: YYYY-Www
|
|
188
|
+
try:
|
|
189
|
+
if "-W" in period:
|
|
190
|
+
year, week = period.split("-W", 1)
|
|
191
|
+
period_labels.append(f"Week {week}, {year}")
|
|
192
|
+
else:
|
|
193
|
+
period_labels.append(period)
|
|
194
|
+
except (ValueError, IndexError):
|
|
195
|
+
period_labels.append(period)
|
|
196
|
+
elif unit == "day":
|
|
197
|
+
period_labels.append(
|
|
198
|
+
datetime.datetime.strptime(period, "%Y-%m-%d").strftime("%b %d, %Y")
|
|
199
|
+
)
|
|
200
|
+
elif unit == "hour":
|
|
201
|
+
period_labels.append(
|
|
202
|
+
datetime.datetime.strptime(period, "%Y-%m-%d-%H").strftime(
|
|
203
|
+
"%b %d, %Y %H:00"
|
|
204
|
+
)
|
|
205
|
+
)
|
|
206
|
+
else:
|
|
207
|
+
period_labels.append(period)
|
|
208
|
+
|
|
209
|
+
# Get experiment data (workspace-level)
|
|
210
|
+
experiment_data = []
|
|
211
|
+
for period in all_periods:
|
|
212
|
+
experiment_count = data.get("experiments_by_unit", {}).get(period, 0)
|
|
213
|
+
experiment_data.append(float(experiment_count) if experiment_count else 0.0)
|
|
214
|
+
|
|
215
|
+
# Get dataset data (workspace-level)
|
|
216
|
+
dataset_data = []
|
|
217
|
+
for period in all_periods:
|
|
218
|
+
dataset_count = data.get("datasets_by_unit", {}).get(period, 0)
|
|
219
|
+
dataset_data.append(float(dataset_count) if dataset_count else 0.0)
|
|
220
|
+
|
|
221
|
+
# Create figure with 6 subplots
|
|
222
|
+
# Increase height to give more room for charts (less space needed for legend now)
|
|
223
|
+
fig, axes = plt.subplots(6, 1, figsize=(14, 20))
|
|
224
|
+
unit_label = unit.capitalize()
|
|
225
|
+
fig.suptitle(
|
|
226
|
+
f'Opik Usage Metrics - {data["workspace"]} (by {unit_label})',
|
|
227
|
+
fontsize=16,
|
|
228
|
+
fontweight="bold",
|
|
229
|
+
)
|
|
230
|
+
|
|
231
|
+
# Chart 1: Trace Count - use top projects only
|
|
232
|
+
ax1 = axes[0]
|
|
233
|
+
x = range(n_periods)
|
|
234
|
+
width = 0.8
|
|
235
|
+
|
|
236
|
+
# Get top projects for trace count
|
|
237
|
+
top_indices, others_data, trace_labels, trace_colors = _get_top_projects_and_others(
|
|
238
|
+
projects, project_names, trace_data, top_n=18
|
|
239
|
+
)
|
|
240
|
+
|
|
241
|
+
bottom = [0] * n_periods
|
|
242
|
+
for idx, (project_idx, label) in enumerate(
|
|
243
|
+
zip(top_indices, trace_labels[: len(top_indices)])
|
|
244
|
+
):
|
|
245
|
+
values: List[float] = [trace_data[j][project_idx] for j in range(n_periods)]
|
|
246
|
+
ax1.bar(x, values, width, label=label, bottom=bottom, color=trace_colors[idx])
|
|
247
|
+
bottom = [float(bottom[j] + values[j]) for j in range(n_periods)] # type: ignore[misc]
|
|
248
|
+
|
|
249
|
+
# Add "Others" if present
|
|
250
|
+
if others_data and any(v > 0 for v in others_data):
|
|
251
|
+
ax1.bar(
|
|
252
|
+
x,
|
|
253
|
+
others_data,
|
|
254
|
+
width,
|
|
255
|
+
label=trace_labels[-1],
|
|
256
|
+
bottom=bottom,
|
|
257
|
+
color=trace_colors[-1],
|
|
258
|
+
)
|
|
259
|
+
|
|
260
|
+
ax1.set_xlabel(unit_label)
|
|
261
|
+
ax1.set_ylabel("Trace Count")
|
|
262
|
+
ax1.set_title(f"Trace Count by {unit_label} (Stacked by Project)")
|
|
263
|
+
ax1.set_xticks(x)
|
|
264
|
+
ax1.set_xticklabels(period_labels, rotation=45, ha="right")
|
|
265
|
+
ax1.legend(
|
|
266
|
+
bbox_to_anchor=(0.5, -0.20),
|
|
267
|
+
loc="upper center",
|
|
268
|
+
ncol=4,
|
|
269
|
+
fontsize=7,
|
|
270
|
+
frameon=True,
|
|
271
|
+
)
|
|
272
|
+
ax1.grid(axis="y", alpha=0.3)
|
|
273
|
+
|
|
274
|
+
# Chart 2: Token Count - use top projects only
|
|
275
|
+
ax2 = axes[1]
|
|
276
|
+
|
|
277
|
+
# Get top projects for token count
|
|
278
|
+
top_indices, others_data, token_labels, token_colors = _get_top_projects_and_others(
|
|
279
|
+
projects, project_names, token_data, top_n=18
|
|
280
|
+
)
|
|
281
|
+
|
|
282
|
+
bottom = [0] * n_periods
|
|
283
|
+
for idx, (project_idx, label) in enumerate(
|
|
284
|
+
zip(top_indices, token_labels[: len(top_indices)])
|
|
285
|
+
):
|
|
286
|
+
values: List[float] = [token_data[j][project_idx] for j in range(n_periods)] # type: ignore[no-redef]
|
|
287
|
+
ax2.bar(x, values, width, label=label, bottom=bottom, color=token_colors[idx])
|
|
288
|
+
bottom = [float(bottom[j] + values[j]) for j in range(n_periods)] # type: ignore[misc]
|
|
289
|
+
|
|
290
|
+
# Add "Others" if present
|
|
291
|
+
if others_data and any(v > 0 for v in others_data):
|
|
292
|
+
ax2.bar(
|
|
293
|
+
x,
|
|
294
|
+
others_data,
|
|
295
|
+
width,
|
|
296
|
+
label=token_labels[-1],
|
|
297
|
+
bottom=bottom,
|
|
298
|
+
color=token_colors[-1],
|
|
299
|
+
)
|
|
300
|
+
|
|
301
|
+
ax2.set_xlabel(unit_label)
|
|
302
|
+
ax2.set_ylabel("Token Count")
|
|
303
|
+
ax2.set_title(f"Token Count by {unit_label} (Stacked by Project)")
|
|
304
|
+
ax2.set_xticks(x)
|
|
305
|
+
ax2.set_xticklabels(period_labels, rotation=45, ha="right")
|
|
306
|
+
ax2.legend(
|
|
307
|
+
bbox_to_anchor=(0.5, -0.20),
|
|
308
|
+
loc="upper center",
|
|
309
|
+
ncol=4,
|
|
310
|
+
fontsize=7,
|
|
311
|
+
frameon=True,
|
|
312
|
+
)
|
|
313
|
+
ax2.grid(axis="y", alpha=0.3)
|
|
314
|
+
# Format y-axis to show in thousands/millions
|
|
315
|
+
ax2.yaxis.set_major_formatter(
|
|
316
|
+
FuncFormatter(
|
|
317
|
+
lambda x, p: (
|
|
318
|
+
f"{x/1e6:.2f}M"
|
|
319
|
+
if x >= 1e6
|
|
320
|
+
else f"{x/1e3:.0f}K"
|
|
321
|
+
if x >= 1e3
|
|
322
|
+
else f"{x:.0f}"
|
|
323
|
+
)
|
|
324
|
+
)
|
|
325
|
+
)
|
|
326
|
+
|
|
327
|
+
# Chart 3: Cost - use top projects only
|
|
328
|
+
ax3 = axes[2]
|
|
329
|
+
|
|
330
|
+
# Get top projects for cost
|
|
331
|
+
top_indices, others_data, cost_labels, cost_colors = _get_top_projects_and_others(
|
|
332
|
+
projects, project_names, cost_data, top_n=18
|
|
333
|
+
)
|
|
334
|
+
|
|
335
|
+
bottom = [0] * n_periods
|
|
336
|
+
for idx, (project_idx, label) in enumerate(
|
|
337
|
+
zip(top_indices, cost_labels[: len(top_indices)])
|
|
338
|
+
):
|
|
339
|
+
values: List[float] = [cost_data[j][project_idx] for j in range(n_periods)] # type: ignore[no-redef]
|
|
340
|
+
ax3.bar(x, values, width, label=label, bottom=bottom, color=cost_colors[idx])
|
|
341
|
+
bottom = [float(bottom[j] + values[j]) for j in range(n_periods)] # type: ignore[misc]
|
|
342
|
+
|
|
343
|
+
# Add "Others" if present
|
|
344
|
+
if others_data and any(v > 0 for v in others_data):
|
|
345
|
+
ax3.bar(
|
|
346
|
+
x,
|
|
347
|
+
others_data,
|
|
348
|
+
width,
|
|
349
|
+
label=cost_labels[-1],
|
|
350
|
+
bottom=bottom,
|
|
351
|
+
color=cost_colors[-1],
|
|
352
|
+
)
|
|
353
|
+
|
|
354
|
+
ax3.set_xlabel(unit_label)
|
|
355
|
+
ax3.set_ylabel("Cost ($)")
|
|
356
|
+
ax3.set_title(f"Cost by {unit_label} (Stacked by Project)")
|
|
357
|
+
ax3.set_xticks(x)
|
|
358
|
+
ax3.set_xticklabels(period_labels, rotation=45, ha="right")
|
|
359
|
+
ax3.legend(
|
|
360
|
+
bbox_to_anchor=(0.5, -0.20),
|
|
361
|
+
loc="upper center",
|
|
362
|
+
ncol=4,
|
|
363
|
+
fontsize=7,
|
|
364
|
+
frameon=True,
|
|
365
|
+
)
|
|
366
|
+
ax3.grid(axis="y", alpha=0.3)
|
|
367
|
+
# Format y-axis for currency
|
|
368
|
+
ax3.yaxis.set_major_formatter(FuncFormatter(lambda x, p: f"${x:.2f}"))
|
|
369
|
+
|
|
370
|
+
# Chart 4: Experiment Count (workspace-level, not stacked)
|
|
371
|
+
ax4 = axes[3]
|
|
372
|
+
ax4.bar(x, experiment_data, width, color="steelblue", alpha=0.7)
|
|
373
|
+
|
|
374
|
+
ax4.set_xlabel(unit_label)
|
|
375
|
+
ax4.set_ylabel("Experiment Count")
|
|
376
|
+
ax4.set_title(f"Experiment Count by {unit_label} (Workspace Total)")
|
|
377
|
+
ax4.set_xticks(x)
|
|
378
|
+
ax4.set_xticklabels(period_labels, rotation=45, ha="right")
|
|
379
|
+
ax4.grid(axis="y", alpha=0.3)
|
|
380
|
+
|
|
381
|
+
# Chart 5: Dataset Count (workspace-level, not stacked)
|
|
382
|
+
ax5 = axes[4]
|
|
383
|
+
ax5.bar(x, dataset_data, width, color="darkgreen", alpha=0.7)
|
|
384
|
+
|
|
385
|
+
ax5.set_xlabel(unit_label)
|
|
386
|
+
ax5.set_ylabel("Dataset Count")
|
|
387
|
+
ax5.set_title(f"Dataset Count by {unit_label} (Workspace Total)")
|
|
388
|
+
ax5.set_xticks(x)
|
|
389
|
+
ax5.set_xticklabels(period_labels, rotation=45, ha="right")
|
|
390
|
+
ax5.grid(axis="y", alpha=0.3)
|
|
391
|
+
|
|
392
|
+
# Chart 6: Span Count - use top projects only
|
|
393
|
+
ax6 = axes[5]
|
|
394
|
+
|
|
395
|
+
# Get top projects for span count
|
|
396
|
+
top_indices, others_data, span_labels, span_colors = _get_top_projects_and_others(
|
|
397
|
+
projects, project_names, span_data, top_n=18
|
|
398
|
+
)
|
|
399
|
+
|
|
400
|
+
bottom = [0] * n_periods
|
|
401
|
+
for idx, (project_idx, label) in enumerate(
|
|
402
|
+
zip(top_indices, span_labels[: len(top_indices)])
|
|
403
|
+
):
|
|
404
|
+
values: List[float] = [span_data[j][project_idx] for j in range(n_periods)] # type: ignore[no-redef]
|
|
405
|
+
ax6.bar(x, values, width, label=label, bottom=bottom, color=span_colors[idx])
|
|
406
|
+
bottom = [float(bottom[j] + values[j]) for j in range(n_periods)] # type: ignore[misc]
|
|
407
|
+
|
|
408
|
+
# Add "Others" if present
|
|
409
|
+
if others_data and any(v > 0 for v in others_data):
|
|
410
|
+
ax6.bar(
|
|
411
|
+
x,
|
|
412
|
+
others_data,
|
|
413
|
+
width,
|
|
414
|
+
label=span_labels[-1],
|
|
415
|
+
bottom=bottom,
|
|
416
|
+
color=span_colors[-1],
|
|
417
|
+
)
|
|
418
|
+
|
|
419
|
+
ax6.set_xlabel(unit_label)
|
|
420
|
+
ax6.set_ylabel("Span Count")
|
|
421
|
+
ax6.set_title(f"Span Count by {unit_label} (Stacked by Project)")
|
|
422
|
+
ax6.set_xticks(x)
|
|
423
|
+
ax6.set_xticklabels(period_labels, rotation=45, ha="right")
|
|
424
|
+
ax6.legend(
|
|
425
|
+
bbox_to_anchor=(0.5, -0.20),
|
|
426
|
+
loc="upper center",
|
|
427
|
+
ncol=4,
|
|
428
|
+
fontsize=7,
|
|
429
|
+
frameon=True,
|
|
430
|
+
)
|
|
431
|
+
ax6.grid(axis="y", alpha=0.3)
|
|
432
|
+
|
|
433
|
+
# Use rect parameter to make room for legends below charts (more space for lower legends)
|
|
434
|
+
plt.tight_layout(rect=[0, 0.0, 1, 0.98])
|
|
435
|
+
|
|
436
|
+
plt.close()
|
|
437
|
+
|
|
438
|
+
|
|
439
|
+
def create_individual_chart(
|
|
440
|
+
data: Dict[str, Any],
|
|
441
|
+
chart_type: str,
|
|
442
|
+
output_dir: str = ".",
|
|
443
|
+
) -> Optional[str]:
|
|
444
|
+
"""
|
|
445
|
+
Create an individual chart figure for a specific chart type.
|
|
446
|
+
|
|
447
|
+
Args:
|
|
448
|
+
data: The extracted data dictionary
|
|
449
|
+
chart_type: Type of chart - "trace_count", "token_count", "cost", "experiment_count", "dataset_count", "span_count"
|
|
450
|
+
output_dir: Directory to save chart (default: current directory)
|
|
451
|
+
|
|
452
|
+
Returns:
|
|
453
|
+
Path to saved chart image file, or None if creation failed
|
|
454
|
+
"""
|
|
455
|
+
try:
|
|
456
|
+
import matplotlib.pyplot as plt
|
|
457
|
+
from matplotlib.ticker import FuncFormatter
|
|
458
|
+
except ImportError:
|
|
459
|
+
raise ImportError(
|
|
460
|
+
"matplotlib is required for chart generation. "
|
|
461
|
+
"Please install it with: pip install matplotlib"
|
|
462
|
+
)
|
|
463
|
+
|
|
464
|
+
# Get unit from data (default to month for backward compatibility)
|
|
465
|
+
unit = data.get("unit", "month")
|
|
466
|
+
|
|
467
|
+
# Prepare data for charts
|
|
468
|
+
projects = [
|
|
469
|
+
p for p in data["projects"] if "metrics_by_unit" in p and "error" not in p
|
|
470
|
+
]
|
|
471
|
+
if not projects:
|
|
472
|
+
return None
|
|
473
|
+
|
|
474
|
+
# Collect all time periods across all projects
|
|
475
|
+
all_periods_set = set()
|
|
476
|
+
for project in projects:
|
|
477
|
+
all_periods_set.update(project["metrics_by_unit"].keys())
|
|
478
|
+
all_periods: List[str] = sorted(all_periods_set)
|
|
479
|
+
|
|
480
|
+
if not all_periods:
|
|
481
|
+
return None
|
|
482
|
+
|
|
483
|
+
# Prepare data arrays for each metric
|
|
484
|
+
project_names = [p["project_name"] for p in projects]
|
|
485
|
+
n_periods = len(all_periods)
|
|
486
|
+
|
|
487
|
+
# Format period labels for display based on unit
|
|
488
|
+
period_labels = []
|
|
489
|
+
for period in all_periods:
|
|
490
|
+
if unit == "month":
|
|
491
|
+
period_labels.append(
|
|
492
|
+
datetime.datetime.strptime(period, "%Y-%m").strftime("%b %Y")
|
|
493
|
+
)
|
|
494
|
+
elif unit == "week":
|
|
495
|
+
try:
|
|
496
|
+
if "-W" in period:
|
|
497
|
+
year, week = period.split("-W", 1)
|
|
498
|
+
period_labels.append(f"Week {week}, {year}")
|
|
499
|
+
else:
|
|
500
|
+
period_labels.append(period)
|
|
501
|
+
except (ValueError, IndexError):
|
|
502
|
+
period_labels.append(period)
|
|
503
|
+
elif unit == "day":
|
|
504
|
+
period_labels.append(
|
|
505
|
+
datetime.datetime.strptime(period, "%Y-%m-%d").strftime("%b %d, %Y")
|
|
506
|
+
)
|
|
507
|
+
elif unit == "hour":
|
|
508
|
+
period_labels.append(
|
|
509
|
+
datetime.datetime.strptime(period, "%Y-%m-%d-%H").strftime(
|
|
510
|
+
"%b %d, %Y %H:00"
|
|
511
|
+
)
|
|
512
|
+
)
|
|
513
|
+
else:
|
|
514
|
+
period_labels.append(period)
|
|
515
|
+
|
|
516
|
+
# Create figure with consistent size for all charts (same as reference implementation)
|
|
517
|
+
fig, ax = plt.subplots(figsize=(14, 8))
|
|
518
|
+
unit_label = unit.capitalize()
|
|
519
|
+
x = range(n_periods)
|
|
520
|
+
width = 0.8
|
|
521
|
+
|
|
522
|
+
if chart_type == "trace_count":
|
|
523
|
+
# Trace count data
|
|
524
|
+
trace_data = extract_metric_data(projects, all_periods, "trace_count")
|
|
525
|
+
|
|
526
|
+
# Get top projects for trace count
|
|
527
|
+
top_indices, others_data, labels, colors = _get_top_projects_and_others(
|
|
528
|
+
projects, project_names, trace_data, top_n=18
|
|
529
|
+
)
|
|
530
|
+
|
|
531
|
+
bottom = [0] * n_periods
|
|
532
|
+
for idx, (project_idx, label) in enumerate(
|
|
533
|
+
zip(top_indices, labels[: len(top_indices)])
|
|
534
|
+
):
|
|
535
|
+
values: List[float] = [trace_data[j][project_idx] for j in range(n_periods)]
|
|
536
|
+
ax.bar(x, values, width, label=label, bottom=bottom, color=colors[idx])
|
|
537
|
+
bottom = [float(bottom[j] + values[j]) for j in range(n_periods)] # type: ignore[misc]
|
|
538
|
+
|
|
539
|
+
# Add "Others" if present
|
|
540
|
+
if others_data and any(v > 0 for v in others_data):
|
|
541
|
+
ax.bar(
|
|
542
|
+
x, others_data, width, label=labels[-1], bottom=bottom, color=colors[-1]
|
|
543
|
+
)
|
|
544
|
+
|
|
545
|
+
ax.set_ylabel("Trace Count")
|
|
546
|
+
ax.set_title(f"Trace Count by {unit_label} (Stacked by Project)")
|
|
547
|
+
|
|
548
|
+
elif chart_type == "token_count":
|
|
549
|
+
# Helper function for token count aggregation
|
|
550
|
+
def aggregate_token_count(token_count: Any) -> float:
|
|
551
|
+
"""Aggregate token count: use total_tokens if available, otherwise sum all values."""
|
|
552
|
+
if isinstance(token_count, dict):
|
|
553
|
+
if "total_tokens" in token_count:
|
|
554
|
+
return float(token_count["total_tokens"])
|
|
555
|
+
else:
|
|
556
|
+
return (
|
|
557
|
+
sum(float(v) for v in token_count.values())
|
|
558
|
+
if token_count
|
|
559
|
+
else 0.0
|
|
560
|
+
)
|
|
561
|
+
else:
|
|
562
|
+
return float(token_count) if token_count else 0.0
|
|
563
|
+
|
|
564
|
+
# Token count data
|
|
565
|
+
token_data = extract_metric_data(
|
|
566
|
+
projects, all_periods, "token_count", aggregate_token_count
|
|
567
|
+
)
|
|
568
|
+
|
|
569
|
+
# Get top projects for token count
|
|
570
|
+
top_indices, others_data, labels, colors = _get_top_projects_and_others(
|
|
571
|
+
projects, project_names, token_data, top_n=18
|
|
572
|
+
)
|
|
573
|
+
|
|
574
|
+
bottom = [0] * n_periods
|
|
575
|
+
for idx, (project_idx, label) in enumerate(
|
|
576
|
+
zip(top_indices, labels[: len(top_indices)])
|
|
577
|
+
):
|
|
578
|
+
values: List[float] = [token_data[j][project_idx] for j in range(n_periods)] # type: ignore[no-redef]
|
|
579
|
+
ax.bar(x, values, width, label=label, bottom=bottom, color=colors[idx])
|
|
580
|
+
bottom = [float(bottom[j] + values[j]) for j in range(n_periods)] # type: ignore[misc]
|
|
581
|
+
|
|
582
|
+
# Add "Others" if present
|
|
583
|
+
if others_data and any(v > 0 for v in others_data):
|
|
584
|
+
ax.bar(
|
|
585
|
+
x, others_data, width, label=labels[-1], bottom=bottom, color=colors[-1]
|
|
586
|
+
)
|
|
587
|
+
|
|
588
|
+
ax.set_ylabel("Token Count")
|
|
589
|
+
ax.set_title(f"Token Count by {unit_label} (Stacked by Project)")
|
|
590
|
+
ax.yaxis.set_major_formatter(
|
|
591
|
+
FuncFormatter(
|
|
592
|
+
lambda x, p: (
|
|
593
|
+
f"{x/1e6:.2f}M"
|
|
594
|
+
if x >= 1e6
|
|
595
|
+
else f"{x/1e3:.0f}K"
|
|
596
|
+
if x >= 1e3
|
|
597
|
+
else f"{x:.0f}"
|
|
598
|
+
)
|
|
599
|
+
)
|
|
600
|
+
)
|
|
601
|
+
|
|
602
|
+
elif chart_type == "cost":
|
|
603
|
+
# Cost data
|
|
604
|
+
cost_data = extract_metric_data(projects, all_periods, "cost")
|
|
605
|
+
|
|
606
|
+
# Get top projects for cost
|
|
607
|
+
top_indices, others_data, labels, colors = _get_top_projects_and_others(
|
|
608
|
+
projects, project_names, cost_data, top_n=18
|
|
609
|
+
)
|
|
610
|
+
|
|
611
|
+
bottom = [0] * n_periods
|
|
612
|
+
for idx, (project_idx, label) in enumerate(
|
|
613
|
+
zip(top_indices, labels[: len(top_indices)])
|
|
614
|
+
):
|
|
615
|
+
values: List[float] = [cost_data[j][project_idx] for j in range(n_periods)] # type: ignore[no-redef]
|
|
616
|
+
ax.bar(x, values, width, label=label, bottom=bottom, color=colors[idx])
|
|
617
|
+
bottom = [float(bottom[j] + values[j]) for j in range(n_periods)] # type: ignore[misc]
|
|
618
|
+
|
|
619
|
+
# Add "Others" if present
|
|
620
|
+
if others_data and any(v > 0 for v in others_data):
|
|
621
|
+
ax.bar(
|
|
622
|
+
x, others_data, width, label=labels[-1], bottom=bottom, color=colors[-1]
|
|
623
|
+
)
|
|
624
|
+
|
|
625
|
+
ax.set_ylabel("Cost ($)")
|
|
626
|
+
ax.set_title(f"Cost by {unit_label} (Stacked by Project)")
|
|
627
|
+
ax.yaxis.set_major_formatter(FuncFormatter(lambda x, p: f"${x:.2f}"))
|
|
628
|
+
|
|
629
|
+
elif chart_type == "experiment_count":
|
|
630
|
+
# Experiment data (workspace-level)
|
|
631
|
+
experiment_data = []
|
|
632
|
+
for period in all_periods:
|
|
633
|
+
experiment_count = data.get("experiments_by_unit", {}).get(period, 0)
|
|
634
|
+
experiment_data.append(float(experiment_count) if experiment_count else 0.0)
|
|
635
|
+
|
|
636
|
+
ax.bar(x, experiment_data, width, color="steelblue", alpha=0.7)
|
|
637
|
+
ax.set_ylabel("Experiment Count")
|
|
638
|
+
ax.set_title(f"Experiment Count by {unit_label} (Workspace Total)")
|
|
639
|
+
|
|
640
|
+
elif chart_type == "dataset_count":
|
|
641
|
+
# Dataset data (workspace-level)
|
|
642
|
+
dataset_data = []
|
|
643
|
+
for period in all_periods:
|
|
644
|
+
dataset_count = data.get("datasets_by_unit", {}).get(period, 0)
|
|
645
|
+
dataset_data.append(float(dataset_count) if dataset_count else 0.0)
|
|
646
|
+
|
|
647
|
+
ax.bar(x, dataset_data, width, color="darkgreen", alpha=0.7)
|
|
648
|
+
ax.set_ylabel("Dataset Count")
|
|
649
|
+
ax.set_title(f"Dataset Count by {unit_label} (Workspace Total)")
|
|
650
|
+
|
|
651
|
+
elif chart_type == "span_count":
|
|
652
|
+
# Span count data
|
|
653
|
+
span_data = extract_metric_data(projects, all_periods, "span_count")
|
|
654
|
+
|
|
655
|
+
# Get top projects for span count
|
|
656
|
+
top_indices, others_data, labels, colors = _get_top_projects_and_others(
|
|
657
|
+
projects, project_names, span_data, top_n=18
|
|
658
|
+
)
|
|
659
|
+
|
|
660
|
+
bottom = [0] * n_periods
|
|
661
|
+
for idx, (project_idx, label) in enumerate(
|
|
662
|
+
zip(top_indices, labels[: len(top_indices)])
|
|
663
|
+
):
|
|
664
|
+
values: List[float] = [span_data[j][project_idx] for j in range(n_periods)] # type: ignore[no-redef]
|
|
665
|
+
ax.bar(x, values, width, label=label, bottom=bottom, color=colors[idx])
|
|
666
|
+
bottom = [float(bottom[j] + values[j]) for j in range(n_periods)] # type: ignore[misc]
|
|
667
|
+
|
|
668
|
+
# Add "Others" if present
|
|
669
|
+
if others_data and any(v > 0 for v in others_data):
|
|
670
|
+
ax.bar(
|
|
671
|
+
x, others_data, width, label=labels[-1], bottom=bottom, color=colors[-1]
|
|
672
|
+
)
|
|
673
|
+
|
|
674
|
+
ax.set_ylabel("Span Count")
|
|
675
|
+
ax.set_title(f"Span Count by {unit_label} (Stacked by Project)")
|
|
676
|
+
|
|
677
|
+
else:
|
|
678
|
+
plt.close()
|
|
679
|
+
return None
|
|
680
|
+
|
|
681
|
+
ax.set_xlabel(unit_label)
|
|
682
|
+
ax.set_xticks(x)
|
|
683
|
+
ax.set_xticklabels(period_labels, rotation=45, ha="right")
|
|
684
|
+
# Set x-axis limits to use full width, with small padding on edges
|
|
685
|
+
ax.set_xlim(-0.5, n_periods - 0.5)
|
|
686
|
+
|
|
687
|
+
ax.grid(axis="y", alpha=0.3)
|
|
688
|
+
|
|
689
|
+
# Configure legend for charts that need it - place inside figure bounds
|
|
690
|
+
has_legend = chart_type in ["trace_count", "token_count", "cost", "span_count"]
|
|
691
|
+
if has_legend:
|
|
692
|
+
# Truncate legend labels to maximum length to prevent overly wide legends
|
|
693
|
+
handles, labels = ax.get_legend_handles_labels()
|
|
694
|
+
max_label_length = 40 # Maximum characters per legend label
|
|
695
|
+
truncated_labels = []
|
|
696
|
+
for label in labels:
|
|
697
|
+
if len(label) > max_label_length:
|
|
698
|
+
truncated_labels.append(label[: max_label_length - 3] + "...")
|
|
699
|
+
else:
|
|
700
|
+
truncated_labels.append(label)
|
|
701
|
+
|
|
702
|
+
# Position legend inside the plot area at the bottom, with more space below
|
|
703
|
+
# This allows us to use bbox_inches=None for fixed image sizes
|
|
704
|
+
# Use 3 columns to ensure items wrap into multiple rows
|
|
705
|
+
ax.legend(
|
|
706
|
+
handles,
|
|
707
|
+
truncated_labels,
|
|
708
|
+
loc="upper center",
|
|
709
|
+
bbox_to_anchor=(0.5, -0.35), # Lower in plot area, ~1.5 inches below chart
|
|
710
|
+
ncol=3, # 3 columns ensures wrapping into multiple rows
|
|
711
|
+
fontsize=8,
|
|
712
|
+
framealpha=0.9,
|
|
713
|
+
)
|
|
714
|
+
|
|
715
|
+
# Explicitly set margins to ensure chart uses full width consistently
|
|
716
|
+
# Left margin (10%) accounts for y-axis labels (including formatted labels like "500.00M" or "$350.00")
|
|
717
|
+
# Right margin (5%) is minimal to maximize chart width
|
|
718
|
+
# Bottom margin (42.5%) accommodates legend positioned below the plot area (outside axes bounds) with ~1 inch of space below
|
|
719
|
+
# Top margin (8%) for title
|
|
720
|
+
# This ensures ALL charts have identical dimensions regardless of y-axis formatter
|
|
721
|
+
fig.subplots_adjust(left=0.10, right=0.95, top=0.92, bottom=0.425)
|
|
722
|
+
|
|
723
|
+
# Save chart to temporary file (use absolute path)
|
|
724
|
+
chart_filename = os.path.join(
|
|
725
|
+
output_dir, f"opik_chart_{chart_type}_{data['workspace']}.png"
|
|
726
|
+
)
|
|
727
|
+
chart_filename = os.path.abspath(chart_filename)
|
|
728
|
+
|
|
729
|
+
# Ensure output directory exists
|
|
730
|
+
chart_dir = os.path.dirname(chart_filename)
|
|
731
|
+
if chart_dir and not os.path.exists(chart_dir):
|
|
732
|
+
os.makedirs(chart_dir, exist_ok=True)
|
|
733
|
+
|
|
734
|
+
try:
|
|
735
|
+
# Use bbox_inches=None to preserve exact figure size (14x8 inches)
|
|
736
|
+
# Since legend is now inside figure bounds, we can use fixed dimensions
|
|
737
|
+
# This ensures ALL charts have identical dimensions (4200x2400 pixels at 300 DPI)
|
|
738
|
+
# regardless of y-axis label widths or content
|
|
739
|
+
plt.savefig(chart_filename, dpi=300, bbox_inches=None)
|
|
740
|
+
plt.close()
|
|
741
|
+
|
|
742
|
+
# Ensure file is fully written to disk using file system sync operations
|
|
743
|
+
# Retry loop to handle cases where file system hasn't fully flushed
|
|
744
|
+
max_retries = 10
|
|
745
|
+
retry_delay = 0.1
|
|
746
|
+
file_ready = False
|
|
747
|
+
|
|
748
|
+
for attempt in range(max_retries):
|
|
749
|
+
if os.path.exists(chart_filename):
|
|
750
|
+
try:
|
|
751
|
+
# Try to open the file to ensure it's accessible
|
|
752
|
+
with open(chart_filename, "rb") as f:
|
|
753
|
+
# Force file system sync
|
|
754
|
+
f.flush()
|
|
755
|
+
os.fsync(f.fileno())
|
|
756
|
+
|
|
757
|
+
# Verify file has content (size > 0)
|
|
758
|
+
if os.path.getsize(chart_filename) > 0:
|
|
759
|
+
# Verify file is readable
|
|
760
|
+
if os.access(chart_filename, os.R_OK):
|
|
761
|
+
file_ready = True
|
|
762
|
+
break
|
|
763
|
+
except (OSError, IOError):
|
|
764
|
+
# File may still be writing, wait and retry
|
|
765
|
+
pass
|
|
766
|
+
|
|
767
|
+
if attempt < max_retries - 1:
|
|
768
|
+
time.sleep(retry_delay)
|
|
769
|
+
|
|
770
|
+
if not file_ready:
|
|
771
|
+
console.print(
|
|
772
|
+
f"[yellow]Warning: Chart file was not ready after {max_retries} attempts: {chart_filename}[/yellow]"
|
|
773
|
+
)
|
|
774
|
+
return None
|
|
775
|
+
|
|
776
|
+
return chart_filename
|
|
777
|
+
except Exception as e:
|
|
778
|
+
plt.close()
|
|
779
|
+
console.print(
|
|
780
|
+
f"[yellow]Warning: Could not save chart {chart_type}: {e}[/yellow]"
|
|
781
|
+
)
|
|
782
|
+
traceback.print_exc()
|
|
783
|
+
return None
|