forecasting-tools 0.2.87__tar.gz → 0.2.88__tar.gz

{forecasting_tools-0.2.87 → forecasting_tools-0.2.88}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: forecasting-tools
-Version: 0.2.87
+Version: 0.2.88
 Summary: AI forecasting and research tools to help humans reason about and forecast the future
 License: MIT
 License-File: LICENSE

{forecasting_tools-0.2.87 → forecasting_tools-0.2.88}/forecasting_tools/agents_and_tools/other/data_analyzer.py

@@ -12,7 +12,6 @@ from forecasting_tools.ai_models.agent_wrappers import (
     CodingTool,
     agent_tool,
     event_to_tool_message,
-    general_trace_or_span,
 )
 from forecasting_tools.util.misc import clean_indents
 
@@ -119,10 +118,9 @@ class DataAnalyzer:
 
 
 if __name__ == "__main__":
-    with general_trace_or_span("Test Span 1"):
-        answer = asyncio.run(
-            DataAnalyzer().run_data_analysis(
-                instructions="Please multiple 52.675 x 6.547 x 9867.5476 x 4356.5",
-            )
+    answer = asyncio.run(
+        DataAnalyzer().run_data_analysis(
+            instructions="Please multiple 52.675 x 6.547 x 9867.5476 x 4356.5",
         )
+    )
     print(answer)

{forecasting_tools-0.2.87 → forecasting_tools-0.2.88}/forecasting_tools/agents_and_tools/research/key_factors_researcher.py

@@ -222,7 +222,7 @@ class KeyFactorsResearcher:
             # Grading Scale for {ScoreCardGrade.__class__.__name__}
             - {ScoreCardGrade.VERY_BAD.value}: Generally poor quality
             - {ScoreCardGrade.BAD.value}: Below average quality
-            - {ScoreCardGrade.OK.value}: Below average quality
+            - {ScoreCardGrade.OK.value}: Average quality
             - {ScoreCardGrade.GOOD.value}: Above average quality
             - {ScoreCardGrade.VERY_GOOD.value}: Exceptional quality
 

forecasting_tools-0.2.88/forecasting_tools/ai_models/agent_wrappers.py

@@ -0,0 +1,75 @@
+import asyncio
+import logging
+
+import nest_asyncio
+from agents import Agent, CodeInterpreterTool, FunctionTool, Runner
+from agents import function_tool as ft
+from agents.extensions.models.litellm_model import LitellmModel
+from agents.stream_events import StreamEvent
+
+from forecasting_tools.ai_models.model_tracker import ModelTracker
+
+logger = logging.getLogger(__name__)
+
+
+nest_asyncio.apply()
+
+
+class AgentSdkLlm(LitellmModel):
+    """
+    Wrapper around openai-agent-sdk's LiteLlm Model for later extension
+    """
+
+    async def get_response(self, *args, **kwargs):  # NOSONAR
+        ModelTracker.give_cost_tracking_warning_if_needed(self.model)
+        response = await super().get_response(*args, **kwargs)
+        await asyncio.sleep(
+            0.0001
+        )  # For whatever reason, it seems you need to await a coroutine to get the litellm cost callback to work
+        return response
+
+
+AgentRunner = Runner  # Alias for Runner for later extension
+AgentTool = FunctionTool  # Alias for FunctionTool for later extension
+AiAgent = Agent  # Alias for Agent for later extension
+CodingTool = CodeInterpreterTool  # Alias for CodeInterpreterTool for later extension
+agent_tool = ft  # Alias for function_tool for later extension
+
+
+def event_to_tool_message(event: StreamEvent) -> str | None:
+    text = ""
+    if event.type == "run_item_stream_event":
+        item = event.item
+        if item.type == "message_output_item":
+            content = item.raw_item.content[0]
+            if content.type == "output_text":
+                # text = content.text
+                text = ""  # the text is already streamed separate from this function
+            elif content.type == "output_refusal":
+                text = content.refusal
+            else:
+                text = "Error: unknown content type"
+        elif item.type == "tool_call_item":
+            if item.raw_item.type == "code_interpreter_call":
+                text = (
+                    f"\nCode interpreter code:\n```python\n{item.raw_item.code}\n```\n"
+                )
+            else:
+                tool_name = getattr(item.raw_item, "name", "unknown_tool")
+                tool_args = getattr(item.raw_item, "arguments", {})
+                text = f"Tool call: {tool_name}({tool_args})"
+        elif item.type == "tool_call_output_item":
+            output = getattr(item, "output", str(item.raw_item))
+            text = f"Tool output:\n\n{output}"
+        elif item.type == "handoff_call_item":
+            handoff_info = getattr(item.raw_item, "name", "handoff")
+            text = f"Handoff call: {handoff_info}"
+        elif item.type == "handoff_output_item":
+            text = f"Handoff output: {str(item.raw_item)}"
+        elif item.type == "reasoning_item":
+            text = f"Reasoning: {str(item.raw_item)}"
+    # elif event.type == "agent_updated_stream_event":
+    #     text += f"Agent updated: {event.new_agent.name}\n\n"
+    if text == "":
+        return None
+    return text

{forecasting_tools-0.2.87 → forecasting_tools-0.2.88}/forecasting_tools/ai_models/general_llm.py

@@ -21,7 +21,6 @@ from openai.types.responses import (
     ResponseReasoningItem,
 )
 
-from forecasting_tools.ai_models.agent_wrappers import track_generation
 from forecasting_tools.ai_models.ai_utils.openai_utils import (
     OpenAiUtils,
     VisionMessageData,
@@ -251,16 +250,7 @@ class GeneralLlm(
     ) -> Any:
         logger.debug(f"Invoking model with prompt: {prompt}")
 
-        with track_generation(
-            input=self.model_input_to_message(prompt),
-            model=self.model,
-        ) as span:
-            direct_call_response = await self._mockable_direct_call_to_model(prompt)
-            answer = direct_call_response.data
-            span.span_data.output = [{"role": "assistant", "content": answer}]
-            # span.span_data.usage = usage.model_dump()
-            span.span_data.model = self.model
-            span.span_data.model_config = self.litellm_kwargs
+        direct_call_response = await self._mockable_direct_call_to_model(prompt)
 
         logger.debug(f"Model responded with: {direct_call_response}")
         return direct_call_response

{forecasting_tools-0.2.87 → forecasting_tools-0.2.88}/forecasting_tools/auto_optimizers/prompt_optimizer.py

@@ -9,12 +9,7 @@ from pydantic import BaseModel, Field
 from forecasting_tools.agents_and_tools.minor_tools import (
     perplexity_reasoning_pro_search,
 )
-from forecasting_tools.ai_models.agent_wrappers import (
-    AgentRunner,
-    AgentSdkLlm,
-    AiAgent,
-    general_trace_or_span,
-)
+from forecasting_tools.ai_models.agent_wrappers import AgentRunner, AgentSdkLlm, AiAgent
 from forecasting_tools.auto_optimizers.prompt_data_models import PromptIdea
 from forecasting_tools.helpers.structure_output import structure_output
 from forecasting_tools.util.misc import clean_indents, retry_async_function
@@ -112,63 +107,57 @@ class PromptOptimizer:
             )
 
     async def create_optimized_prompt(self) -> OptimizationRun:
-        with general_trace_or_span("Prompt Optimizer"):
-            return await self._create_optimized_prompt()
+        return await self._create_optimized_prompt()
 
     async def _create_optimized_prompt(self) -> OptimizationRun:
         iteration_num = 0
-        with general_trace_or_span("Initial Population Generation (Iteration 1)"):
-            iteration_num += 1
-            logger.info(
-                f"Generating initial prompt population of size {self.initial_prompt_population_size}"
-            )
-            seed_prompt = ImplementedPrompt(
-                text=self.initial_prompt,
-                idea=PromptIdea(
-                    short_name="Initial Seed",
-                    full_text="The user-provided initial prompt",
-                ),
-                originating_ideas=[],
-            )
-            starting_prompts: list[ImplementedPrompt] = [seed_prompt]
-            prompts_still_needed = self.initial_prompt_population_size - len(
-                starting_prompts
-            )
-            if prompts_still_needed > 0:
-                additional_initial_prompts = await self._mutate_prompt(
-                    starting_prompts[0],
-                    prompts_still_needed,
-                )
-                starting_prompts.extend(additional_initial_prompts)
-
-            offspring_prompts: list[ImplementedPrompt] = starting_prompts
-            assert (
-                seed_prompt in offspring_prompts
-            ), "Seed prompt not found in offspring prompts"
-            all_evaluated_prompts: list[ScoredPrompt] = (
-                await self._evaluate_new_members(offspring_prompts)
+        iteration_num += 1
+        logger.info(
+            f"Generating initial prompt population of size {self.initial_prompt_population_size}"
+        )
+        seed_prompt = ImplementedPrompt(
+            text=self.initial_prompt,
+            idea=PromptIdea(
+                short_name="Initial Seed",
+                full_text="The user-provided initial prompt",
+            ),
+            originating_ideas=[],
+        )
+        starting_prompts: list[ImplementedPrompt] = [seed_prompt]
+        prompts_still_needed = self.initial_prompt_population_size - len(
+            starting_prompts
+        )
+        if prompts_still_needed > 0:
+            additional_initial_prompts = await self._mutate_prompt(
+                starting_prompts[0],
+                prompts_still_needed,
             )
-            survivors = await self._kill_the_weak(all_evaluated_prompts)
+            starting_prompts.extend(additional_initial_prompts)
+
+        offspring_prompts: list[ImplementedPrompt] = starting_prompts
+        assert (
+            seed_prompt in offspring_prompts
+        ), "Seed prompt not found in offspring prompts"
+        all_evaluated_prompts: list[ScoredPrompt] = await self._evaluate_new_members(
+            offspring_prompts
+        )
+        survivors = await self._kill_the_weak(all_evaluated_prompts)
 
         while iteration_num < self.iterations:
             iteration_num += 1
-            with general_trace_or_span(
-                f"Prompt Optimizer Iteration {iteration_num + 1}",
-                data={"survivors": [s.model_dump() for s in survivors]},
-            ):
-                logger.info(
-                    f"Starting iteration {iteration_num + 1}/{self.iterations} - Current population size: {len(offspring_prompts)}"
-                )
+            logger.info(
+                f"Starting iteration {iteration_num + 1}/{self.iterations} - Current population size: {len(offspring_prompts)}"
+            )
 
-                offspring_prompts = await self._generate_new_prompts(survivors)
+            offspring_prompts = await self._generate_new_prompts(survivors)
 
-                evaluated_prompts = await self._evaluate_new_members(offspring_prompts)
-                all_evaluated_prompts.extend(evaluated_prompts)
-                updated_population = survivors + evaluated_prompts
+            evaluated_prompts = await self._evaluate_new_members(offspring_prompts)
+            all_evaluated_prompts.extend(evaluated_prompts)
+            updated_population = survivors + evaluated_prompts
 
-                survivors = await self._kill_the_weak(updated_population)
+            survivors = await self._kill_the_weak(updated_population)
 
-                self._log_duplicate_prompts(all_evaluated_prompts)
+            self._log_duplicate_prompts(all_evaluated_prompts)
 
         return OptimizationRun(scored_prompts=all_evaluated_prompts)
 

{forecasting_tools-0.2.87 → forecasting_tools-0.2.88}/forecasting_tools/cp_benchmarking/benchmarker.py

@@ -5,7 +5,6 @@ from typing import Sequence
 
 import typeguard
 
-from forecasting_tools.ai_models.agent_wrappers import general_trace_or_span
 from forecasting_tools.ai_models.resource_managers.monetary_cost_manager import (
     MonetaryCostManager,
 )
@@ -80,50 +79,44 @@ class Benchmarker:
         self.code_to_snapshot = additional_code_to_snapshot
 
     async def run_benchmark(self) -> list[BenchmarkForBot]:
-        with general_trace_or_span("Benchmarker"):
-            if self.questions_to_use is None:
-                assert (
-                    self.number_of_questions_to_use is not None
-                ), "number_of_questions_to_use must be provided if questions_to_use is not provided"
-                chosen_questions = MetaculusApi.get_benchmark_questions(
-                    self.number_of_questions_to_use,
-                )
-            else:
-                chosen_questions = self.questions_to_use
-
-            chosen_questions = typeguard.check_type(
-                chosen_questions, list[MetaculusQuestion]
-            )
-
-            if self.number_of_questions_to_use is not None:
-                assert len(chosen_questions) == self.number_of_questions_to_use
-
-            benchmarks: list[BenchmarkForBot] = self._initialize_benchmarks(
-                self.forecast_bots, chosen_questions
+        if self.questions_to_use is None:
+            assert (
+                self.number_of_questions_to_use is not None
+            ), "number_of_questions_to_use must be provided if questions_to_use is not provided"
+            chosen_questions = MetaculusApi.get_benchmark_questions(
+                self.number_of_questions_to_use,
             )
-
-            batches = self._batch_questions(
-                self.forecast_bots,
-                benchmarks,
-                chosen_questions,
-                self.concurrent_question_batch_size,
+        else:
+            chosen_questions = self.questions_to_use
+
+        chosen_questions = typeguard.check_type(
+            chosen_questions, list[MetaculusQuestion]
+        )
+
+        if self.number_of_questions_to_use is not None:
+            assert len(chosen_questions) == self.number_of_questions_to_use
+
+        benchmarks: list[BenchmarkForBot] = self._initialize_benchmarks(
+            self.forecast_bots, chosen_questions
+        )
+
+        batches = self._batch_questions(
+            self.forecast_bots,
+            benchmarks,
+            chosen_questions,
+            self.concurrent_question_batch_size,
+        )
+        try:
+            for i, batch in enumerate(batches):
+                await self._run_a_batch(batch)
+                if batch.is_last_batch_for_benchmark:
+                    self._append_benchmarks_to_jsonl_if_configured([batch.benchmark])
+        except KeyboardInterrupt:
+            logger.warning(
+                "KeyboardInterrupt detected, saving current benchmark progress."
             )
-            try:
-                for i, batch in enumerate(batches):
-                    with general_trace_or_span(
-                        f"{batch.benchmark.name} - Batch {i+1} of {len(batches)}"
-                    ):
-                        await self._run_a_batch(batch)
-                    if batch.is_last_batch_for_benchmark:
-                        self._append_benchmarks_to_jsonl_if_configured(
-                            [batch.benchmark]
-                        )
-            except KeyboardInterrupt:
-                logger.warning(
-                    "KeyboardInterrupt detected, saving current benchmark progress."
-                )
-                self._append_benchmarks_to_jsonl_if_configured([batch.benchmark])
-                raise
+            self._append_benchmarks_to_jsonl_if_configured([batch.benchmark])
+            raise
         return benchmarks
 
     async def _run_a_batch(self, batch: QuestionBatch) -> None:

{forecasting_tools-0.2.87 → forecasting_tools-0.2.88}/forecasting_tools/data_models/numeric_report.py

@@ -314,6 +314,22 @@ class NumericDistribution(BaseModel):
         ]
         return representative_percentiles
 
+    def get_percentiles_at_target_heights(
+        self, target_heights: list[float] | None = None
+    ) -> list[Percentile]:
+        if target_heights is None:
+            target_heights = [0.1, 0.2, 0.4, 0.6, 0.8, 0.9]
+
+        heights = [p.percentile for p in self.declared_percentiles]
+        values = [p.value for p in self.declared_percentiles]
+
+        result = []
+        for target in target_heights:
+            interpolated_value = float(np.interp(target, heights, values))
+            result.append(Percentile(percentile=target, value=interpolated_value))
+
+        return result
+
     @property
     @typing_extensions.deprecated(
         "NumericDistribution.cdf (property) will be replaced with NumericDistribution.get_cdf (method). Please switch.",
@@ -633,12 +649,9 @@ class NumericReport(ForecastReport):
     def make_readable_prediction(cls, prediction: NumericDistribution) -> str:
         num_percentiles = len(prediction.declared_percentiles)
         if num_percentiles > 10:
-            num_display_percentiles = 5
+            representative_percentiles = prediction.get_percentiles_at_target_heights()
         else:
-            num_display_percentiles = num_percentiles
-        representative_percentiles = prediction.get_representative_percentiles(
-            num_display_percentiles
-        )
+            representative_percentiles = prediction.declared_percentiles
         readable = "Probability distribution:\n"
         for percentile in representative_percentiles:
             if prediction.is_date:

{forecasting_tools-0.2.87 → forecasting_tools-0.2.88}/forecasting_tools/forecast_bots/forecast_bot.py

@@ -12,7 +12,6 @@ from typing import Any, Coroutine, Literal, Sequence, TypeVar, cast, overload
 from exceptiongroup import ExceptionGroup
 from pydantic import BaseModel
 
-from forecasting_tools.ai_models.agent_wrappers import general_trace_or_span
 from forecasting_tools.ai_models.general_llm import GeneralLlm
 from forecasting_tools.ai_models.resource_managers.monetary_cost_manager import (
     MonetaryCostManager,
@@ -338,20 +337,17 @@ class ForecastBot(ABC):
     async def _run_individual_question_with_error_propagation(
         self, question: MetaculusQuestion
     ) -> ForecastReport:
-        with general_trace_or_span(
-            f"{self.__class__.__name__} - Question: {question.page_url}"
-        ):
-            try:
-                return await self._run_individual_question(question)
-            except Exception as e:
-                error_message = (
-                    f"Error while processing question url: '{question.page_url}'"
-                )
-                logger.error(f"{error_message}: {e}")
-                self._reraise_exception_with_prepended_message(e, error_message)
-                assert (
-                    False
-                ), "This is to satisfy type checker. The previous function should raise an exception"
+        try:
+            return await self._run_individual_question(question)
+        except Exception as e:
+            error_message = (
+                f"Error while processing question url: '{question.page_url}'"
+            )
+            logger.error(f"{error_message}: {e}")
+            self._reraise_exception_with_prepended_message(e, error_message)
+            assert (
+                False
+            ), "This is to satisfy type checker. The previous function should raise an exception"
 
     async def _run_individual_question(
         self, question: MetaculusQuestion

{forecasting_tools-0.2.87 → forecasting_tools-0.2.88}/forecasting_tools/front_end/app_pages/chat_page.py

@@ -43,7 +43,6 @@ from forecasting_tools.ai_models.agent_wrappers import (
     AgentTool,
     AiAgent,
     event_to_tool_message,
-    general_trace_or_span,
 )
 from forecasting_tools.ai_models.resource_managers.monetary_cost_manager import (
     MonetaryCostManager,
@@ -84,7 +83,6 @@ class ChatSession(BaseModel, Jsonable):
     name: str
     messages: list[dict]
     model_choice: str = DEFAULT_MODEL
-    trace_id: str | None = None
     last_chat_cost: float | None = None
     last_chat_duration: float | None = None
     time_stamp: datetime = Field(default_factory=datetime.now)
@@ -235,11 +233,6 @@ class ChatPage(AppPage):
                 st.markdown(
                     f"**Last Chat Duration:** {st.session_state.last_chat_duration:.2f} seconds"
                 )
-            if "trace_id" in st.session_state.keys():
-                trace_id = st.session_state.trace_id
-                st.markdown(
-                    f"**Conversation in Foresight Project:** [link](https://platform.openai.com/traces/trace?trace_id={trace_id})"
-                )
 
     @classmethod
     def display_premade_examples(cls) -> None:
@@ -265,8 +258,6 @@ class ChatPage(AppPage):
                     if st.button(session.name, key=session.name):
                         st.session_state.messages = session.messages
                         st.session_state.model_choice = session.model_choice
-                        if session.trace_id:
-                            st.session_state.trace_id = session.trace_id
                         if session.last_chat_cost:
                             st.session_state.last_chat_cost = session.last_chat_cost
                         if session.last_chat_duration:
@@ -299,7 +290,6 @@ class ChatPage(AppPage):
                         name=st.session_state["chat_save_name"],
                         model_choice=st.session_state["model_choice"],
                         messages=st.session_state.messages,
-                        trace_id=st.session_state.trace_id,
                         last_chat_cost=st.session_state.last_chat_cost,
                         last_chat_duration=st.session_state.last_chat_duration,
                     )
@@ -464,29 +454,27 @@ class ChatPage(AppPage):
             handoffs=[],
         )
 
-        with general_trace_or_span("Chat App") as chat_trace:
-            result = AgentRunner.run_streamed(
-                agent, st.session_state.messages, max_turns=20
-            )
-            streamed_text = ""
-            with st.chat_message("assistant"):
-                placeholder = st.empty()
-            with st.spinner("Thinking..."):
-                async for event in result.stream_events():
-                    if event.type == "raw_response_event" and isinstance(
-                        event.data, ResponseTextDeltaEvent
-                    ):
-                        streamed_text += event.data.delta
-                    placeholder.write(streamed_text)
-
-                    new_reasoning = event_to_tool_message(event)
-                    if new_reasoning:
-                        st.sidebar.write(new_reasoning)
-
-            # logger.info(f"Chat finished with output: {streamed_text}")
-            st.session_state.messages = result.to_input_list()
-            st.session_state.trace_id = chat_trace.trace_id
-            cls._update_last_message_if_gemini_bug(model_choice)
+        result = AgentRunner.run_streamed(
+            agent, st.session_state.messages, max_turns=20
+        )
+        streamed_text = ""
+        with st.chat_message("assistant"):
+            placeholder = st.empty()
+        with st.spinner("Thinking..."):
+            async for event in result.stream_events():
+                if event.type == "raw_response_event" and isinstance(
+                    event.data, ResponseTextDeltaEvent
+                ):
+                    streamed_text += event.data.delta
+                placeholder.write(streamed_text)
+
+                new_reasoning = event_to_tool_message(event)
+                if new_reasoning:
+                    st.sidebar.write(new_reasoning)
+
+        # logger.info(f"Chat finished with output: {streamed_text}")
+        st.session_state.messages = result.to_input_list()
+        cls._update_last_message_if_gemini_bug(model_choice)
 
         ForecastDatabaseManager.add_general_report_to_database(
             question_text=prompt_input,
@@ -519,7 +507,6 @@ class ChatPage(AppPage):
     @classmethod
     def clear_chat_history(cls) -> None:
         st.session_state.messages = [cls.DEFAULT_MESSAGE]
-        st.session_state.trace_id = None
         st.session_state.chat_files = []
 
 

{forecasting_tools-0.2.87 → forecasting_tools-0.2.88}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "forecasting-tools"
-version = "0.2.87"
+version = "0.2.88"
 description = "AI forecasting and research tools to help humans reason about and forecast the future"
 authors = ["Benjamin Wilson <ben@metaculus.com>"]
 license = "MIT"