relai 0.3.3__py3-none-any.whl → 0.3.5__py3-none-any.whl

relai/maestro/optimizer.py

@@ -134,8 +134,7 @@ class Maestro:
         """
         self.total_visits += 1
         self.versions[self.current_version]["average_score"] = (
-            self.versions[self.current_version]["average_score"] * self.versions[self.current_version]["visits"]
-            + score
+            self.versions[self.current_version]["average_score"] * self.versions[self.current_version]["visits"] + score
         ) / (self.versions[self.current_version]["visits"] + 1.0)
         self.versions[self.current_version]["visits"] += 1
 
@@ -161,7 +160,7 @@ class Maestro:
             return str(agent_outputs)
 
     async def _evaluate(
-        self, awaitables: list[Awaitable], criticos: list[Critico], verbose: bool = False, print_flag: str = ""
+        self, awaitables: list[Awaitable], criticos: list[Critico], verbose: bool = True, print_flag: str = ""
     ) -> tuple[list[dict[str, Any]], list[AgentLog]]:
         """
         Run and evaluate the current version of the agent through a set of awaitables.
@@ -170,7 +169,7 @@ class Maestro:
             awaitables (list[Awaitable]): A list of awaitables, each representing a run of the agent
             criticos (list[Critico]): A list of Critico objects, each corresponding to an awaitable
             verbose (bool): If True, additional information will be printed during evaluation.
-                Defaults to False.
+                Defaults to True.
             print_flag (str): A string to be put next to the printed info when `verbose` is True.
                 Used to distinguish printed info from different types of evaluations.
 
@@ -228,7 +227,7 @@ class Maestro:
         self,
         batch_size: int,
         sampler: ProportionalSampler,
-        verbose: bool = False,
+        verbose: bool = True,
         group_id: str | None = None,
         pbar: tqdm | None = None,
     ) -> bool:
@@ -245,7 +244,7 @@ class Maestro:
                 `batch_size` of them will be used for preliminary examinations.
             sampler (ProportionalSampler): Sampler to use for selecting setups.
             verbose (bool): If True, additional information will be printed during the iterate step.
-                Defaults to False.
+                Defaults to True.
             group_id (str, optional): An optional group ID to associate all runs together. If not provided,
                 a new UUID will be generated.
             pbar (tqdm, optional): A progress bar to display the progress of the iteration. Defaults to None.
@@ -374,17 +373,17 @@ class Maestro:
     async def optimize_config(
         self,
         total_rollouts: int,
-        batch_size: int = 4,
+        batch_size: int = 8,
         explore_radius: int = 5,
         explore_factor: float = 0.5,
-        verbose: bool = False,
+        verbose: bool = True,
     ):
         """
         Optimize the configs (parameters) of the agent.
 
         Args:
             total_rollouts (int): Total number of rollouts to use for optimization.
-            batch_size (int): Base batch size to use for individual optimization steps. Defaults to 4.
+            batch_size (int): Base batch size to use for individual optimization steps. Defaults to 8.
             explore_radius (int): A positive integer controlling the aggressiveness of exploration during optimization.
                 A larger `explore_radius` encourages the optimizer to make more substantial changes between successive configurations.
                 Defaults to 5.
@@ -393,7 +392,7 @@ class Maestro:
                 while a lower value allocates more rollouts to ensure the discovered configs are thoroughly evaluated.
                 Defaults to 0.5.
             verbose (bool): If True, related information will be printed during the optimization step.
-                Defaults to False.
+                Defaults to True.
 
         Raises:
             ValueError: If the input parameters are not valid.
@@ -407,16 +406,17 @@ class Maestro:
         if explore_factor <= 0 or explore_factor >= 1:
             raise ValueError(f"`explore_factor` must be a float between 0 and 1, got {explore_factor}.")
 
-        # total_rollouts = (iterate_steps * batch_size * 4 + select_steps * batch_size) * num_rounds
-        # explore_factor = (iterate_steps * batch_size * 4) / (iterate_steps * batch_size * 4 + select_steps * batch_size)
+        group_size = (batch_size + 1) // 2
+        # total_rollouts = (iterate_steps * group_size * 4 + select_steps * group_size) * num_rounds
+        # explore_factor = (iterate_steps * group_size * 4) / (iterate_steps * group_size * 4 + select_steps * group_size)
         iterate_steps: int = explore_radius
         select_steps: int = int(explore_radius * 4 * (1 - explore_factor) / explore_factor)
-        num_rounds: int = int(total_rollouts / (iterate_steps * batch_size * 4 + select_steps * batch_size))
-        total_rollouts = num_rounds * (iterate_steps * batch_size * 4 + select_steps * batch_size)
+        num_rounds: int = int(total_rollouts / (iterate_steps * group_size * 4 + select_steps * group_size))
+        total_rollouts = num_rounds * (iterate_steps * group_size * 4 + select_steps * group_size)
 
-        print("optimize_config settings:")
+        print("\n" + "=" * 30 + "optimize_config settings" + "=" * 30)
         print("  total_rollouts: ", total_rollouts)
-        print("  batch_size: ", batch_size)
+        print("  (adjusted) batch_size: ", group_size * 2)
         print("  explore_radius: ", explore_radius)
         print("  explore_factor: ", explore_factor)
         print("-" * 60)
@@ -428,29 +428,33 @@ class Maestro:
         if num_rounds == 0:
             raise ValueError(
                 f"`total_rollouts` is too small for the given `batch_size` {batch_size}, `explore_radius` {explore_radius}, and `explore_factor` {explore_factor}. "
-                f"Please increase `total_rollouts` to at least {iterate_steps * batch_size * 4 + select_steps * batch_size}."
+                f"Please increase `total_rollouts` to at least {iterate_steps * group_size * 4 + select_steps * group_size}."
             )
 
         sampler = ProportionalSampler(
             elements=self.setups,
             weights=[setup["weight"] for setup in self.setups],
         )
-        group_id = uuid4().hex
+        group_id = "Maestro-Config-" + uuid4().hex
         pbar = tqdm(total=total_rollouts, desc="Total rollouts consumed for config optimization")
 
         for round in range(num_rounds):
-            print("=" * 30 + f" Round {round + 1}/{num_rounds} begins" + "=" * 30)
-            print("Total versions: ", len(self.versions))
+            print("\n\n" + "=" * 30 + f" Round {round + 1}/{num_rounds} begins" + "=" * 30)
+            print("Total versions accepted: ", len(self.versions))
             print("Rebase to version: ", self.current_version)
-            print("Score (current base): ", self.versions[self.current_version]["average_score"])
-            print("Visits (current base): ", self.versions[self.current_version]["visits"])
-            print("Visits (total): ", self.total_visits)
+            print(
+                "Score for the current base version: %s based on %s rollouts"
+                % (
+                    self.versions[self.current_version]["average_score"],
+                    self.versions[self.current_version]["visits"] * group_size,
+                )
+            )
             print("\n\n")
 
             new_version = False
             for _ in range(iterate_steps):
                 changes_accepted = await self._iterate(
-                    batch_size=batch_size, verbose=verbose, sampler=sampler, group_id=group_id, pbar=pbar
+                    batch_size=group_size, verbose=verbose, sampler=sampler, group_id=group_id, pbar=pbar
                 )
                 if changes_accepted:
                     new_version = True
@@ -474,7 +478,7 @@ class Maestro:
             for _ in range(select_steps):
                 await self._select(explore=True)
 
-                setups = sampler.sample(batch_size)
+                setups = sampler.sample(group_size)
                 awaitables = []
                 criticos = []
                 for setup in setups:
@@ -518,17 +522,21 @@ class Maestro:
             # Switch to the current version with highest score
             await self._select(explore=False)
 
-            print("=" * 30 + f" Round {round + 1}/{num_rounds} finishes" + "=" * 30)
-            print("Total versions: ", len(self.versions))
-            print("Best version: ", self.current_version)
-            print("Score (best version): ", self.versions[self.current_version]["average_score"])
-            print("Visits (best version): ", self.versions[self.current_version]["visits"])
-            print("Visits (total): ", self.total_visits)
+            print("\n\n" + "=" * 30 + f" Round {round + 1}/{num_rounds} finishes" + "=" * 30)
+            print("Total versions accepted: ", len(self.versions))
+            print("Best version index: ", self.current_version)
+            print(
+                "Score for the best version: %s based on %s rollouts"
+                % (
+                    self.versions[self.current_version]["average_score"],
+                    self.versions[self.current_version]["visits"] * group_size,
+                )
+            )
 
             print(
-                "all versions: ",
+                "All versions: ",
                 {
-                    i: {"score": self.versions[i]["average_score"], "visits": self.versions[i]["visits"]}
+                    i: {"score": self.versions[i]["average_score"], "rollouts evaluated": self.versions[i]["visits"] * group_size}
                     for i in range(len(self.versions))
                 },
             )
@@ -569,7 +577,10 @@ class Maestro:
             if self.log_to_platform:
                 await sync_to_platform()
                 print(
-                    f"Results of round {round + 1}/{num_rounds} uploaded to RELAI platform, visualization id: {self.config_opt_viz_id}"
+                    (
+                        f"Results of round {round + 1}/{num_rounds} uploaded to RELAI platform, "
+                        f"visualization id: {self.config_opt_viz_id}\n\n\n"
+                    )
                 )
 
     async def optimize_structure(
@@ -577,7 +588,7 @@ class Maestro:
         total_rollouts: int,
         description: Optional[str] = None,
         code_paths: Optional[list[str]] = None,
-        verbose: bool = False,
+        verbose: bool = True,
     ) -> str:
         """
         Propose structural changes (i.e. changes that cannot be achieved by setting parameters alone) to
@@ -591,15 +602,15 @@ class Maestro:
             code_paths (list[str], optional): A list of paths corresponding to code files containing
                 the implementation of the agent.
             verbose (bool): If True, additional information will be printed during the optimization.
-                Defaults to False.
+                Defaults to True.
 
         Returns:
             str: Suggestion for structural changes to the agent.
         """
 
-        print("optimize_structure settings:")
+        print("\n" + "=" * 30 + "optimize_structure settings" + "=" * 30)
         print("  total_rollouts: ", total_rollouts)
-        print("-" * 60 + "\n\n")
+        print("=" * 80 + "\n\n")
 
         if code_paths is not None:
             code = extract_code(code_paths=code_paths)
@@ -610,9 +621,10 @@ class Maestro:
             elements=self.setups,
             weights=[setup["weight"] for setup in self.setups],
         )
-        group_id = uuid4().hex
+        group_id = "Maestro-Struct-" + uuid4().hex
 
-        print("Running the agent to collect traces...")
+        print("=" * 80)
+        print("Running the agent to collect traces...\n\n")
 
         setups = sampler.sample(total_rollouts)
         awaitables = []
@@ -625,7 +637,8 @@ class Maestro:
 
         test_cases, _ = await self._evaluate(awaitables=awaitables, criticos=criticos, verbose=verbose)
 
-        print("Optimizing structure...")
+        print("=" * 80)
+        print("Optimizing structure...\n\n")
         suggestion = await self._client.optimize_structure(
             {
                 "agent_name": get_full_func_name(self.agent_fn),
@@ -664,6 +677,6 @@ class Maestro:
 
         if self.log_to_platform:
             uid = await sync_to_platform()
-            print(f"Results uploaded to RELAI platform, visualization id: {uid}")
+            print(f"Results uploaded to RELAI platform, visualization id: {uid}\n\n\n")
 
         return suggestion

relai/mocker/tool.py

@@ -4,6 +4,7 @@ from uuid import uuid4
 
 from agents import Agent, Runner, SQLiteSession
 
+from ..utils import no_trace
 from .base_mocker import BaseMocker
 
 
@@ -48,12 +49,12 @@ class MockTool(BaseMocker):
                 "kwargs": kwargs,
             }
         )
-
-        result = Runner.run_sync(
-            self.agent,
-            agent_input,
-            session=self._session,
-        )
+        with no_trace():
+            result = Runner.run_sync(
+                self.agent,
+                agent_input,
+                session=self._session,
+            )
         output = result.final_output
         return output
 
@@ -64,11 +65,11 @@ class MockTool(BaseMocker):
                 "kwargs": kwargs,
             }
         )
-
-        result = await Runner.run(
-            self.agent,
-            agent_input,
-            session=self._session,
-        )
+        with no_trace():
+            result = await Runner.run(
+                self.agent,
+                agent_input,
+                session=self._session,
+            )
         output = result.final_output
         return output

relai/simulator.py

@@ -213,7 +213,7 @@ class SyncSimulator(BaseSimulator):
                 a new UUID will be generated.
         """
         agent_logs: list[AgentLog] = []
-        group_id = uuid4().hex if group_id is None else group_id
+        group_id = ("Simulate-" + uuid4().hex) if group_id is None else group_id
         tracking_on()
         for tape, config in self.tape_and_config_generator(num_runs):
             with _simulate(config), create_logging_span(tape.id):
@@ -248,7 +248,7 @@ class SyncSimulator(BaseSimulator):
                 a new UUID will be generated.
         """
         agent_logs: list[AgentLog] = []
-        group_id = uuid4().hex if group_id is None else group_id
+        group_id = ("Simulate-" + uuid4().hex) if group_id is None else group_id
         tracking_on()
         for tape in simulation_tapes:
             new_tape = tape.copy()
@@ -312,7 +312,7 @@ class AsyncSimulator(BaseSimulator):
                 a new UUID will be generated.
         """
         agent_logs: list[AgentLog] = []
-        group_id = uuid4().hex if group_id is None else group_id
+        group_id = ("Simulate-" + uuid4().hex) if group_id is None else group_id
         tracking_on()
         for tape, config in self.tape_and_config_generator(num_runs):
             with _simulate(config), create_logging_span(tape.id):
@@ -347,7 +347,7 @@ class AsyncSimulator(BaseSimulator):
                 a new UUID will be generated.
         """
         agent_logs: list[AgentLog] = []
-        group_id = uuid4().hex if group_id is None else group_id
+        group_id = ("Simulate-" + uuid4().hex) if group_id is None else group_id
         tracking_on()
         for tape in simulation_tapes:
             new_tape = tape.copy()

{relai-0.3.3.dist-info → relai-0.3.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: relai
-Version: 0.3.3
+Version: 0.3.5
 Summary: An SDK for building reliable AI agents
 Author-email: RELAI <priyatham@relai.ai>, RELAI <wwx@relai.ai>
 License:                                 Apache License
@@ -205,6 +205,7 @@ License:                                 Apache License
         See the License for the specific language governing permissions and
         limitations under the License.
 Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Development Status :: 4 - Beta
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
@@ -235,7 +236,7 @@ Dynamic: license-file
 </p>
 
 
-**RELAI** is an SDK for building **reliable AI agents**. It streamlines the hardest parts of agent development—**simulation**, **evaluation**, and **optimization**—so you can iterate quickly with confidence.
+**RELAI** is a platform for building **reliable AI agents**. It streamlines the hardest parts of agent development—**simulation**, **evaluation**, and **optimization**—so you can iterate quickly with confidence.
 
 **What you get**
 - **Agent Simulation** — Create full/partial environments, define LLM personas, mock MCP servers & tools, and generate synthetic data. Optionally condition simulation on real samples to better match production.
@@ -314,16 +315,22 @@ register_param(
 
 
 async def agent_fn(tape: SimulationTape) -> AgentOutputs:
-    question = await get_user_query()
-    agent = Agent(
-        name="Stock assistant",
-        instructions=params.prompt,  # access registered parameter
-        model="gpt-5-mini",
-    )
-    result = await Runner.run(agent, question)
-    tape.extras["format_rubrics"] = {"Prices must include cents (eg: $XXX.XX)": 1.0}
-    tape.agent_inputs["question"] = question  # trace inputs for later auditing
-    return {"summary": result.final_output}
+    # It is good practice to catch exceptions in agent function
+    # especially if the agent might raise errors with different configs
+    try:
+        question = await get_user_query()
+        agent = Agent(
+            name="Stock assistant",
+            instructions=params.prompt,  # access registered parameter
+            model="gpt-5-mini",
+        )
+        result = await Runner.run(agent, question)
+        tape.extras["format_rubrics"] = {"Prices must include cents (eg: $XXX.XX)": 1.0}
+        tape.agent_inputs["question"] = question  # trace inputs for later auditing
+        return {"summary": result.final_output}
+    except Exception as e:
+        return {"summary": str(e)}
+    
 
 
 async def main() -> None:
@@ -363,10 +370,10 @@ async def main() -> None:
         # params.load("saved_config.json")  # load previous params if available
         await maestro.optimize_config(
             total_rollouts=20,  # Total number of rollouts to use for optimization.
-            batch_size=1,  # Base batch size to use for individual optimization steps. Defaults to 4.
+            batch_size=2,  # Base batch size to use for individual optimization steps. Defaults to 4.
             explore_radius=1,  # A positive integer controlling the aggressiveness of exploration during optimization.
             explore_factor=0.5,  # A float between 0 to 1 controlling the exploration-exploitation trade-off.
-            verbose=True,  # If True, related information will be printed during the optimization step.
+            verbose=True,  # If True, additional information will be printed during the optimization step.
         )
         params.save("saved_config.json")  # save optimized params for future usage
 
@@ -377,7 +384,7 @@ async def main() -> None:
         await maestro.optimize_structure(
             total_rollouts=10,  # Total number of rollouts to use for optimization.
             code_paths=["stock-assistant.py"],  # A list of paths corresponding to code implementations of the agent.
-            verbose=True,  # If True, related information will be printed during the optimization step.
+            verbose=True,  # If True, additional information will be printed during the optimization step.
         )
 
 
@@ -404,6 +411,7 @@ Maestro is a holistic agent optimizer. It consumes evaluator/user feedback to im
 
 - 📘 **Documentation:** [docs.relai.ai](http://docs.relai.ai)
 - 🧪 **Examples:** [relai-sdk/examples](examples)
+- 📖 **Tutorials:** [docs.relai.ai/tutorials/index.html](https://docs.relai.ai/tutorials/index.html)
 - 🌐 **Website:** [relai.ai](https://relai.ai)
 - 📰 **Maestro Technical Report:** [ArXiV](https://arxiv.org/abs/2509.04642)
 - 🌐 **Join the Community:** [Discord](https://discord.gg/sjaHJ34YYE)

{relai-0.3.3.dist-info → relai-0.3.5.dist-info}/RECORD

@@ -6,23 +6,23 @@ relai/data.py,sha256=ne0H4EQ0B_yxE9fogoovGExuJuwqutSpuhNsl4UmcsU,7852
 relai/exporter.py,sha256=jZxrUjlYCOpRr7gdmbg6-LUL_fXmtMgPp89CgvP5Z7A,1932
 relai/flags.py,sha256=_GrjQg7mZq7BwEIedR6cjWY4grwsryqbKdgyiRr2P7k,1929
 relai/logger.py,sha256=j6PdzNkltukWAqBGKAB2qH2p61kS60RwsupDz-gELB4,18358
-relai/simulator.py,sha256=Ni-RxaCtDfjgYuaYOSRTwVrvxhsX_a76x8Jr1EUEP0M,16401
+relai/simulator.py,sha256=oEC5oLODPo1vLGBaMUdDj0JovZlc595dez931ihDuXk,16465
 relai/utils.py,sha256=va3xz79NTLJiZKaBrS_3Y8dC4M_JEmf8uOwzwFYYqUU,2359
 relai/critico/__init__.py,sha256=c_mDXCVEzsQckDS4ZFOmANo8vB5Vjr1bvyQNimAPVR8,52
 relai/critico/critico.py,sha256=J1ek9v2J5WBnHnZknZEVppIrWGczVHxuRX7ghK6mpXM,7616
 relai/critico/evaluate.py,sha256=Bd-Hlsh2fz2AQ0SINoyqcdpdbWK2t8yrAPHv6UCueFY,31348
 relai/maestro/__init__.py,sha256=NVXy0v7yghGwGbtsPti4gQGtVA3vMgXdpIpiJUesqME,186
 relai/maestro/graph.py,sha256=SyY0rHzes3o5bSqlK66CQDUAeyChUhWJQM3FzJCBvfs,1850
-relai/maestro/optimizer.py,sha256=y_1hxyf1z6YLUTBJn84aZGmxtDX162CCG5W6eHUDJF0,29100
+relai/maestro/optimizer.py,sha256=irefqbtD5qfPt2QOQaQwd8bsC3uhx7HdNF9d3KDHgD4,29549
 relai/maestro/params.py,sha256=-0Dtk23ClHJR6Q-PsaKr-GwUylz0-BIIquJF2eA-p-I,8925
 relai/maestro/utils.py,sha256=WIE3cR8EMDVfAJozEfngh8DfOQdRPZMxxtN-M1cMmxo,7276
 relai/mocker/__init__.py,sha256=JP2xlSG6Szc0tSEiZzCN6UXdE66uy7AmRn-p358xFVM,102
 relai/mocker/base_mocker.py,sha256=BL4WYtdxWHZdKICfo9idW5i5MrkoxJDElcoeGk-jaJM,994
 relai/mocker/persona.py,sha256=q2A_lwYrp7H6sKkguMIPl7FQ_6pL4kTaxGBJ1kU2aGA,6678
-relai/mocker/tool.py,sha256=wgbmOOTlpVClDMWzfuJfsrNwGI99k9CwzjoaRMLkAyo,2112
+relai/mocker/tool.py,sha256=dHXkVcD9D6HMNlBj13V7GTgW_99a_-3tf9rC6iLDFn8,2229
 relai/schema/visual.py,sha256=Y6BP5CHxLU0e7sTfNjgKmG2GD0R9a8rvITusxd-d-UE,2443
-relai-0.3.3.dist-info/licenses/LICENSE.md,sha256=UNo7WT0mbmbUFjRGzRGaBtybmBPB7xd2ls9tfCkv0oc,10979
-relai-0.3.3.dist-info/METADATA,sha256=fhqMNABGwuXAYOUL8vYyEjtD937SfTmJOObwktBrM5M,23125
-relai-0.3.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-relai-0.3.3.dist-info/top_level.txt,sha256=pRyA93fRj-HsukRNHyS4sHdvLO4TY8VvBMK44KcxRA4,6
-relai-0.3.3.dist-info/RECORD,,
+relai-0.3.5.dist-info/licenses/LICENSE.md,sha256=UNo7WT0mbmbUFjRGzRGaBtybmBPB7xd2ls9tfCkv0oc,10979
+relai-0.3.5.dist-info/METADATA,sha256=ZvzLQHneMR6jcqD-taaKkh9BX93xMHX_LL64AqNnM2U,23533
+relai-0.3.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+relai-0.3.5.dist-info/top_level.txt,sha256=pRyA93fRj-HsukRNHyS4sHdvLO4TY8VvBMK44KcxRA4,6
+relai-0.3.5.dist-info/RECORD,,