fasticrl 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fasticrl/__init__.py +4 -0
- fasticrl/icrl_learner.py +166 -0
- fasticrl/learner/__init__.py +0 -0
- fasticrl/learner/core.py +31 -0
- fasticrl/learner/models/learn_output.py +17 -0
- fasticrl/learner/prompts.py +47 -0
- fasticrl/models/__init__.py +0 -0
- fasticrl/models/agent_save_state.py +11 -0
- fasticrl/models/attempt.py +8 -0
- fasticrl/models/sentinel.py +1 -0
- fasticrl/py.typed +0 -0
- fasticrl/reward/__init__.py +0 -0
- fasticrl/reward/core.py +24 -0
- fasticrl/reward/models/reward_output.py +14 -0
- fasticrl/reward/prompts.py +28 -0
- fasticrl/strategist/__init__.py +0 -0
- fasticrl/strategist/core.py +32 -0
- fasticrl/strategist/models/strategy_output.py +10 -0
- fasticrl/strategist/prompts.py +32 -0
- fasticrl-1.0.0.dist-info/METADATA +215 -0
- fasticrl-1.0.0.dist-info/RECORD +22 -0
- fasticrl-1.0.0.dist-info/WHEEL +4 -0
fasticrl/__init__.py
ADDED
fasticrl/icrl_learner.py
ADDED
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
from fasticrl.strategist.models.strategy_output import StrategyOutput
|
|
2
|
+
|
|
3
|
+
import math
|
|
4
|
+
import tqdm
|
|
5
|
+
import yaml
|
|
6
|
+
from agno.models.base import Model
|
|
7
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
8
|
+
from functools import partial
|
|
9
|
+
|
|
10
|
+
import fasticrl.models.sentinel as sentinel
|
|
11
|
+
from fasticrl.learner.core import LearnerAgent
|
|
12
|
+
from fasticrl.learner.models.learn_output import LearnerOutput
|
|
13
|
+
from fasticrl.models.agent_save_state import AgentSaveState
|
|
14
|
+
from fasticrl.models.attempt import Attempt
|
|
15
|
+
from fasticrl.reward.core import RewardAgent
|
|
16
|
+
from fasticrl.reward.models.reward_output import RewardOutput
|
|
17
|
+
from fasticrl.strategist.core import StrategistAgent
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class ICRLLearner:
|
|
21
|
+
def __init__(
|
|
22
|
+
self,
|
|
23
|
+
learner_model: Model,
|
|
24
|
+
reward_model: Model,
|
|
25
|
+
strategy_model: Model,
|
|
26
|
+
task_description: str = sentinel._sentinel,
|
|
27
|
+
tasks: list[str] = None,
|
|
28
|
+
buffer: list[Attempt] = None,
|
|
29
|
+
strategy: str = "",
|
|
30
|
+
):
|
|
31
|
+
if task_description is sentinel._sentinel:
|
|
32
|
+
raise ValueError("task_description must be provided")
|
|
33
|
+
if tasks is not None and len(tasks) == 0:
|
|
34
|
+
raise ValueError("tasks list must not be empty")
|
|
35
|
+
|
|
36
|
+
self.buffer = buffer if buffer is not None else []
|
|
37
|
+
self.task_description = task_description
|
|
38
|
+
self.tasks = tasks if tasks is not None else []
|
|
39
|
+
self.strategy = strategy
|
|
40
|
+
|
|
41
|
+
self.learner_agent = LearnerAgent(model=learner_model)
|
|
42
|
+
self.reward_agent = RewardAgent(model=reward_model)
|
|
43
|
+
self.strategist_agent = StrategistAgent(model=strategy_model)
|
|
44
|
+
|
|
45
|
+
self.episode = len(self.buffer)
|
|
46
|
+
|
|
47
|
+
def auto_learn(
|
|
48
|
+
self,
|
|
49
|
+
episodes: int = None,
|
|
50
|
+
batch_size: int = 1,
|
|
51
|
+
cli_mode: bool = False,
|
|
52
|
+
strategy_update_interval: int = None,
|
|
53
|
+
):
|
|
54
|
+
if not self.tasks:
|
|
55
|
+
raise ValueError("tasks list is empty — nothing to learn from")
|
|
56
|
+
|
|
57
|
+
if episodes is None:
|
|
58
|
+
episodes = math.ceil(len(self.tasks) / batch_size)
|
|
59
|
+
|
|
60
|
+
iterator = tqdm.tqdm(range(episodes)) if cli_mode else range(episodes)
|
|
61
|
+
|
|
62
|
+
for i in iterator:
|
|
63
|
+
batch_tasks = [
|
|
64
|
+
str(self.tasks[(self.episode + j) % len(self.tasks)])
|
|
65
|
+
for j in range(batch_size)
|
|
66
|
+
]
|
|
67
|
+
|
|
68
|
+
buffer_snapshot = self.__attempts_as_xml()
|
|
69
|
+
|
|
70
|
+
if batch_size == 1:
|
|
71
|
+
attempts = [self._generate_attempt_for_task(batch_tasks[0], buffer_snapshot)]
|
|
72
|
+
else:
|
|
73
|
+
with ThreadPoolExecutor(max_workers=batch_size) as pool:
|
|
74
|
+
attempts = list(pool.map(
|
|
75
|
+
partial(self._generate_attempt_for_task, buffer_as_xml=buffer_snapshot),
|
|
76
|
+
batch_tasks,
|
|
77
|
+
))
|
|
78
|
+
|
|
79
|
+
self.buffer.extend(attempts)
|
|
80
|
+
self.episode += batch_size
|
|
81
|
+
|
|
82
|
+
if strategy_update_interval is not None and strategy_update_interval > 0 and (i + 1) % strategy_update_interval == 0:
|
|
83
|
+
self.update_strategy()
|
|
84
|
+
|
|
85
|
+
def __attempts_as_xml(self) -> str:
|
|
86
|
+
attempts = ""
|
|
87
|
+
for attempt in self.buffer:
|
|
88
|
+
attempts += (
|
|
89
|
+
"<attempt>\n"
|
|
90
|
+
+ f"\tTask: {attempt.task}\n\tOutput: {attempt.output}\n\tReward: {str(attempt.reward)}"
|
|
91
|
+
+ "\n</attempt>\n"
|
|
92
|
+
)
|
|
93
|
+
return attempts.strip()
|
|
94
|
+
|
|
95
|
+
def generate_action(self, task: str, buffer_as_xml: str = None) -> LearnerOutput:
|
|
96
|
+
return self.learner_agent.generate_learning_output(
|
|
97
|
+
task=task,
|
|
98
|
+
task_description=self.task_description,
|
|
99
|
+
strategy=self.strategy,
|
|
100
|
+
buffer_as_xml=buffer_as_xml if buffer_as_xml is not None else self.__attempts_as_xml(),
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
def generate_reward(self, task: str, action: LearnerOutput) -> RewardOutput:
|
|
104
|
+
return self.reward_agent.generate_reward_output(
|
|
105
|
+
task=task, output=action.learning_output
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
def _generate_attempt_for_task(self, task: str, buffer_as_xml: str = None) -> Attempt:
|
|
109
|
+
learner_output: LearnerOutput = self.generate_action(task, buffer_as_xml=buffer_as_xml)
|
|
110
|
+
reward_output: RewardOutput = self.generate_reward(task, learner_output)
|
|
111
|
+
return Attempt(
|
|
112
|
+
task=task,
|
|
113
|
+
reward=reward_output.reward,
|
|
114
|
+
output=learner_output.learning_output,
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
def generate_attempt_by_present_task(self) -> Attempt:
|
|
118
|
+
return self._generate_attempt_for_task(self._present_learning_task)
|
|
119
|
+
|
|
120
|
+
def update_strategy(self):
|
|
121
|
+
strategist_out: StrategyOutput = self.strategist_agent.generate_new_strategy(
|
|
122
|
+
task_description=self.task_description,
|
|
123
|
+
existing_strategy=self.strategy,
|
|
124
|
+
buffer_as_xml=self.__attempts_as_xml(),
|
|
125
|
+
)
|
|
126
|
+
self.strategy = strategist_out.strategy
|
|
127
|
+
|
|
128
|
+
@property
|
|
129
|
+
def _present_learning_task(self) -> str:
|
|
130
|
+
return str(self.tasks[self.episode % len(self.tasks)])
|
|
131
|
+
|
|
132
|
+
@property
|
|
133
|
+
def agent_save_state(self) -> AgentSaveState:
|
|
134
|
+
return AgentSaveState(
|
|
135
|
+
task_description=self.task_description,
|
|
136
|
+
strategy=self.strategy,
|
|
137
|
+
buffer=[dict(b.model_dump(mode="json")) for b in self.buffer],
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
def to_yaml(self, path: str):
|
|
141
|
+
if not path.endswith(".yaml"):
|
|
142
|
+
path += ".yaml"
|
|
143
|
+
|
|
144
|
+
with open(path, "w") as outfile:
|
|
145
|
+
yaml.dump(
|
|
146
|
+
self.agent_save_state.model_dump(mode="json"),
|
|
147
|
+
outfile,
|
|
148
|
+
default_flow_style=False,
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
@classmethod
|
|
152
|
+
def from_yaml(
|
|
153
|
+
cls, path: str, learner_model: Model, reward_model: Model, strategy_model: Model
|
|
154
|
+
):
|
|
155
|
+
with open(path, "r") as f:
|
|
156
|
+
save_state = yaml.load(f, Loader=yaml.SafeLoader)
|
|
157
|
+
agent_state: AgentSaveState = AgentSaveState.model_validate(save_state)
|
|
158
|
+
|
|
159
|
+
return ICRLLearner(
|
|
160
|
+
task_description=agent_state.task_description,
|
|
161
|
+
buffer=agent_state.buffer,
|
|
162
|
+
strategy=agent_state.strategy,
|
|
163
|
+
learner_model=learner_model,
|
|
164
|
+
reward_model=reward_model,
|
|
165
|
+
strategy_model=strategy_model,
|
|
166
|
+
)
|
|
File without changes
|
fasticrl/learner/core.py
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
from fasticrl.learner.prompts import ICRL_LEARNING_SYSTEM_PROMPT
|
|
2
|
+
from agno.agent import Agent
|
|
3
|
+
from agno.models.base import Model
|
|
4
|
+
from agno.run.agent import RunOutput
|
|
5
|
+
|
|
6
|
+
from fasticrl.learner.models.learn_output import LearnerOutput
|
|
7
|
+
from fasticrl.learner.prompts import ICRL_LEARNING_INPUT_PROMPT
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class LearnerAgent(Agent):
|
|
11
|
+
|
|
12
|
+
def __init__(self, model: Model, **kwargs):
|
|
13
|
+
super().__init__(**kwargs)
|
|
14
|
+
|
|
15
|
+
self.model = model
|
|
16
|
+
self.output_schema = LearnerOutput
|
|
17
|
+
self.system_message = ICRL_LEARNING_SYSTEM_PROMPT
|
|
18
|
+
|
|
19
|
+
def generate_learning_output(
|
|
20
|
+
self, buffer_as_xml: str, task: str, task_description: str, strategy: str
|
|
21
|
+
) -> LearnerOutput:
|
|
22
|
+
run_out: RunOutput = super().run(
|
|
23
|
+
ICRL_LEARNING_INPUT_PROMPT.format(
|
|
24
|
+
attempts=buffer_as_xml,
|
|
25
|
+
task=task,
|
|
26
|
+
task_description=task_description,
|
|
27
|
+
strategy=strategy,
|
|
28
|
+
)
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
return run_out.content
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
from typing import Annotated, Literal
|
|
2
|
+
from pydantic import BaseModel, Field
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class LearnerOutput(BaseModel):
|
|
6
|
+
decision: Annotated[
|
|
7
|
+
Literal["EXPLOIT", "EXPLORE"],
|
|
8
|
+
Field(description="The explore/exploit decision for this step."),
|
|
9
|
+
] # Not used in the code again - just to let the model think about the strategies
|
|
10
|
+
reflection: Annotated[
|
|
11
|
+
str,
|
|
12
|
+
Field(description="Brief justification for the decision."),
|
|
13
|
+
]
|
|
14
|
+
learning_output: Annotated[
|
|
15
|
+
str,
|
|
16
|
+
Field(description="The actual response to the task. No meta-commentary — only the answer."),
|
|
17
|
+
]
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
ICRL_LEARNING_SYSTEM_PROMPT = """
|
|
2
|
+
You are an In-Context Reinforcement Learning (ICRL) agent. Your objective is to maximize cumulative reward over time by learning from feedback and dynamically balancing exploration and exploitation.
|
|
3
|
+
|
|
4
|
+
### Exploration vs. Exploitation Decision
|
|
5
|
+
|
|
6
|
+
Use this decision framework:
|
|
7
|
+
|
|
8
|
+
| Situation | Decision |
|
|
9
|
+
|---|---|
|
|
10
|
+
| Recent rewards are high and stable | EXPLOIT |
|
|
11
|
+
| Recent rewards are declining or stagnant | EXPLORE |
|
|
12
|
+
| Few attempts exist (< 3) | EXPLORE |
|
|
13
|
+
| High reward variance across attempts | EXPLORE |
|
|
14
|
+
| Clear winning strategy identified | EXPLOIT |
|
|
15
|
+
|
|
16
|
+
**Exploit:** Apply the best-known strategy with minor refinements.
|
|
17
|
+
**Explore:** Deliberately try a different approach — vary structure, reasoning style, or output format.
|
|
18
|
+
|
|
19
|
+
### Input Structure (XML):
|
|
20
|
+
- <attempts>: Historical (input, output, reward) triples. Your learning signal.
|
|
21
|
+
- <taskdescription>: The specific sub-task to solve this turn.
|
|
22
|
+
- <strategy>: Accumulated strategy derived from past attempts. Empty on first run.
|
|
23
|
+
|
|
24
|
+
### Core Directives:
|
|
25
|
+
1. Analyze <attempts> to identify what drove high vs. low rewards.
|
|
26
|
+
2. Set `decision` to EXPLOIT or EXPLORE based on the framework above.
|
|
27
|
+
3. Set `reflection` to a brief justification for that decision.
|
|
28
|
+
4. Set `learning_output` to your answer to the task — no meta-commentary, no decision labels, just the answer.
|
|
29
|
+
""".strip()
|
|
30
|
+
|
|
31
|
+
ICRL_LEARNING_INPUT_PROMPT = """
|
|
32
|
+
<attempts>
|
|
33
|
+
{attempts}
|
|
34
|
+
</attempts>
|
|
35
|
+
|
|
36
|
+
<task>
|
|
37
|
+
{task}
|
|
38
|
+
</task>
|
|
39
|
+
|
|
40
|
+
<taskdescription>
|
|
41
|
+
{task_description}
|
|
42
|
+
</taskdescription>
|
|
43
|
+
|
|
44
|
+
<strategy>
|
|
45
|
+
{strategy}
|
|
46
|
+
</strategy>
|
|
47
|
+
""".strip()
|
|
File without changes
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
from typing import Annotated
|
|
2
|
+
|
|
3
|
+
from pydantic import BaseModel, Field
|
|
4
|
+
|
|
5
|
+
from fasticrl.models.attempt import Attempt
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class AgentSaveState(BaseModel):
|
|
9
|
+
task_description: Annotated[str, Field()]
|
|
10
|
+
strategy: Annotated[str, Field(default="")]
|
|
11
|
+
buffer: Annotated[list[Attempt], Field(default_factory=list)]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
_sentinel = object()
|
fasticrl/py.typed
ADDED
|
File without changes
|
|
File without changes
|
fasticrl/reward/core.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
from agno.agent import Agent
|
|
2
|
+
from agno.models.base import Model
|
|
3
|
+
from agno.run.agent import RunOutput
|
|
4
|
+
|
|
5
|
+
from fasticrl.reward.models.reward_output import RewardOutput
|
|
6
|
+
from fasticrl.reward.prompts import ICRL_REWARD_INPUT_PROMPT, ICRL_REWARD_SYSTEM_PROMPT
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class RewardAgent(Agent):
|
|
10
|
+
|
|
11
|
+
def __init__(self, model: Model, **kwargs):
|
|
12
|
+
super().__init__(**kwargs)
|
|
13
|
+
|
|
14
|
+
self.model = model
|
|
15
|
+
self.output_schema = RewardOutput
|
|
16
|
+
self.system_message = ICRL_REWARD_SYSTEM_PROMPT
|
|
17
|
+
|
|
18
|
+
def generate_reward_output(self, task: str, output: str) -> RewardOutput:
|
|
19
|
+
|
|
20
|
+
run_out: RunOutput = super().run(
|
|
21
|
+
ICRL_REWARD_INPUT_PROMPT.format(task=task, output=output)
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
return run_out.content
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
from pydantic import Field
|
|
2
|
+
from typing import Annotated
|
|
3
|
+
from pydantic import BaseModel
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class RewardOutput(BaseModel):
|
|
7
|
+
reward: Annotated[
|
|
8
|
+
int,
|
|
9
|
+
Field(
|
|
10
|
+
description="Reward to the given output. Must be in range from 1 - 10.",
|
|
11
|
+
ge=1,
|
|
12
|
+
le=10,
|
|
13
|
+
),
|
|
14
|
+
]
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
ICRL_REWARD_SYSTEM_PROMPT = """
|
|
2
|
+
You are an expert evaluator acting as the reward function for an In-Context Reinforcement Learning (ICRL) system. Your objective is to critically and objectively score the quality of an AI-generated output based strictly on the provided task.
|
|
3
|
+
|
|
4
|
+
# Evaluation Criteria
|
|
5
|
+
When evaluating, consider the following dimensions:
|
|
6
|
+
- Accuracy & Relevance: Does the output correctly address the core task?
|
|
7
|
+
- Constraint Satisfaction: Were all specific instructions and formats followed?
|
|
8
|
+
- Quality & Clarity: Is the output coherent, well-structured, and logically sound?
|
|
9
|
+
|
|
10
|
+
# Scoring Rubric (0 - 10)
|
|
11
|
+
Use the entire 0-10 scale. Avoid score inflation; do not default to extreme scores unless completely warranted.
|
|
12
|
+
- 0-2 (Fail): Completely incorrect, irrelevant, or fails to address the core task.
|
|
13
|
+
- 3-4 (Poor): Partially addresses the task but contains major omissions, errors, or hallucinations.
|
|
14
|
+
- 5-6 (Average): Acceptable. Meets the basic requirements but lacks polish, nuance, or is merely mediocre. Use this baseline frequently.
|
|
15
|
+
- 7-8 (Good): Accurate, well-crafted, and follows instructions closely with only minor flaws.
|
|
16
|
+
- 9-10 (Exceptional): Flawless execution. Perfectly hits all constraints with high-quality reasoning and formatting.
|
|
17
|
+
""".strip()
|
|
18
|
+
|
|
19
|
+
ICRL_REWARD_INPUT_PROMPT = """
|
|
20
|
+
# Input
|
|
21
|
+
<task>
|
|
22
|
+
{task}
|
|
23
|
+
</task>
|
|
24
|
+
|
|
25
|
+
<output>
|
|
26
|
+
{output}
|
|
27
|
+
</output>
|
|
28
|
+
""".strip()
|
|
File without changes
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
from agno.agent import Agent
|
|
2
|
+
from agno.models.base import Model
|
|
3
|
+
from agno.run.agent import RunOutput
|
|
4
|
+
|
|
5
|
+
from fasticrl.strategist.models.strategy_output import StrategyOutput
|
|
6
|
+
from fasticrl.strategist.prompts import (
|
|
7
|
+
ICRL_STRATEGY_INPUT_PROMPT,
|
|
8
|
+
ICRL_STRATEGY_SYSTEM_PROMPT,
|
|
9
|
+
)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class StrategistAgent(Agent):
|
|
13
|
+
|
|
14
|
+
def __init__(self, model: Model, **kwargs):
|
|
15
|
+
super().__init__(**kwargs)
|
|
16
|
+
|
|
17
|
+
self.model = model
|
|
18
|
+
self.system_message = ICRL_STRATEGY_SYSTEM_PROMPT
|
|
19
|
+
self.output_schema = StrategyOutput
|
|
20
|
+
|
|
21
|
+
def generate_new_strategy(
|
|
22
|
+
self, task_description: str, existing_strategy: str, buffer_as_xml: str
|
|
23
|
+
) -> StrategyOutput:
|
|
24
|
+
run_out: RunOutput = super().run(
|
|
25
|
+
ICRL_STRATEGY_INPUT_PROMPT.format(
|
|
26
|
+
task_description=task_description,
|
|
27
|
+
strategy=existing_strategy,
|
|
28
|
+
attempts=buffer_as_xml,
|
|
29
|
+
)
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
return run_out.content
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
ICRL_STRATEGY_SYSTEM_PROMPT = """
|
|
2
|
+
You are an expert Meta-Cognitive Analyst and Prompt Engineer specializing in In-Context Reinforcement Learning. Your sole objective is to analyze execution history and synthesize actionable strategies that maximize future rewards.
|
|
3
|
+
|
|
4
|
+
### Role
|
|
5
|
+
You do NOT solve tasks. You analyze attempts, identify reward-driving patterns, and produce refined strategies for the agent that does.
|
|
6
|
+
|
|
7
|
+
### Analysis Framework
|
|
8
|
+
When evaluating attempts, apply these lenses in order:
|
|
9
|
+
|
|
10
|
+
1. **Pattern Recognition:** What specific behaviors in high-reward (8-10) outputs differ from low-reward (0-4) outputs? Look for structure, tone, detail level, constraint adherence.
|
|
11
|
+
2. **Failure Mode Classification:** Categorize low-reward causes:
|
|
12
|
+
- Formatting violations
|
|
13
|
+
- Insufficient detail or hallucination
|
|
14
|
+
- Constraint/instruction ignored
|
|
15
|
+
- Wrong reasoning approach
|
|
16
|
+
3. **Incremental Refinement:** Never discard an existing strategy. Extend it with new evidence — add concrete Do's and Don'ts.
|
|
17
|
+
4. **Actionability:** Every strategy element must be a directive a model can directly follow, not a vague observation.
|
|
18
|
+
""".strip()
|
|
19
|
+
|
|
20
|
+
ICRL_STRATEGY_INPUT_PROMPT = """
|
|
21
|
+
<task_description>
|
|
22
|
+
{task_description}
|
|
23
|
+
</task_description>
|
|
24
|
+
|
|
25
|
+
<existing_strategy>
|
|
26
|
+
{strategy}
|
|
27
|
+
</existing_strategy>
|
|
28
|
+
|
|
29
|
+
<attempts>
|
|
30
|
+
{attempts}
|
|
31
|
+
</attempts>
|
|
32
|
+
""".strip()
|
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: fasticrl
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: In-Context Reinforcement Learning framework for LLMs — no fine-tuning required.
|
|
5
|
+
Keywords: reinforcement-learning,in-context-learning,icrl,llm,ai,agents,prompt-engineering
|
|
6
|
+
Author: Maximilian König
|
|
7
|
+
Author-email: Maximilian König <maximilian.koenig@mein.gmx>
|
|
8
|
+
Classifier: Development Status :: 3 - Alpha
|
|
9
|
+
Classifier: Intended Audience :: Developers
|
|
10
|
+
Classifier: Intended Audience :: Science/Research
|
|
11
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
12
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
16
|
+
Requires-Dist: agno>=2.6.5
|
|
17
|
+
Requires-Dist: pyyaml>=6.0.3
|
|
18
|
+
Requires-Dist: tqdm>=4.67.3
|
|
19
|
+
Requires-Dist: ollama>=0.6.2 ; extra == 'ollama'
|
|
20
|
+
Requires-Dist: openai>=2.32.0 ; extra == 'openai'
|
|
21
|
+
Requires-Python: >=3.13
|
|
22
|
+
Project-URL: Homepage, https://github.com/makoeta/FastICRL
|
|
23
|
+
Project-URL: Issues, https://github.com/makoeta/FastICRL/issues
|
|
24
|
+
Project-URL: Source, https://github.com/makoeta/FastICRL
|
|
25
|
+
Provides-Extra: ollama
|
|
26
|
+
Provides-Extra: openai
|
|
27
|
+
Description-Content-Type: text/markdown
|
|
28
|
+
|
|
29
|
+
# FastICRL
|
|
30
|
+
|
|
31
|
+
<p align="center">
|
|
32
|
+
<a href="https://pypi.org/project/fasticrl/"><img src="https://img.shields.io/pypi/v/fasticrl.svg" alt="PyPI version"></a>
|
|
33
|
+
<a href="https://pypi.org/project/fasticrl/"><img src="https://img.shields.io/pypi/pyversions/fasticrl.svg" alt="Python"></a>
|
|
34
|
+
<a href="LICENSE"><img src="https://img.shields.io/badge/License-MIT-yellow.svg" alt="License: MIT"></a>
|
|
35
|
+
<img src="https://img.shields.io/badge/no%20fine--tuning-required-brightgreen" alt="No fine-tuning required">
|
|
36
|
+
<img src="https://img.shields.io/badge/no%20GPU-required-blue" alt="No GPU required">
|
|
37
|
+
<a href="https://github.com/agno-agi/agno"><img src="https://img.shields.io/badge/powered%20by-agno-8A2BE2" alt="Powered by agno"></a>
|
|
38
|
+
</p>
|
|
39
|
+
|
|
40
|
+
**In-Context Reinforcement Learning for LLMs — no fine-tuning, no gradient updates, no GPU.**
|
|
41
|
+
|
|
42
|
+
FastICRL implements the ICRL paradigm from [*Reward Is Enough: LLMs Are In-Context Reinforcement Learners*](https://arxiv.org/abs/2506.06303) (Song et al., 2025). A learner LLM improves its outputs purely by reading its own history of attempts and rewards inside the context window — guided by a meta-cognitive strategist. No training, no infrastructure, just inference.
|
|
43
|
+
|
|
44
|
+
---
|
|
45
|
+
|
|
46
|
+
## How it works
|
|
47
|
+
|
|
48
|
+
Three LLM agents collaborate in a feedback loop:
|
|
49
|
+
|
|
50
|
+
```text
|
|
51
|
+
┌──────────────────────────────────────────────────┐
|
|
52
|
+
│ ICRLLearner │
|
|
53
|
+
│ │
|
|
54
|
+
│ Task ──► Learner ──► Output ──► Reward Agent │
|
|
55
|
+
│ ▲ │ │
|
|
56
|
+
│ │ Attempt │ │
|
|
57
|
+
│ │ (task, output, score) │ │
|
|
58
|
+
│ └─────────────────────────┘ │
|
|
59
|
+
│ │ │
|
|
60
|
+
│ (every N episodes) │
|
|
61
|
+
│ ▼ │
|
|
62
|
+
│ Strategist │
|
|
63
|
+
│ (refines the strategy) │
|
|
64
|
+
└──────────────────────────────────────────────────┘
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
| Agent | Role |
|
|
68
|
+
| --- | --- |
|
|
69
|
+
| **Learner** | Generates task outputs; balances exploration vs. exploitation based on reward history |
|
|
70
|
+
| **Reward** | Scores each output on a 0–10 scale (acts as the reward function) |
|
|
71
|
+
| **Strategist** | Analyzes past attempts to synthesize actionable strategies for future episodes |
|
|
72
|
+
|
|
73
|
+
Each agent can be backed by a different model — e.g. a cheap model for reward, a powerful one for the learner.
|
|
74
|
+
|
|
75
|
+
---
|
|
76
|
+
|
|
77
|
+
## Installation
|
|
78
|
+
|
|
79
|
+
```bash
|
|
80
|
+
pip install fasticrl
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
Or with [uv](https://github.com/astral-sh/uv):
|
|
84
|
+
|
|
85
|
+
```bash
|
|
86
|
+
uv add fasticrl
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
Model provider extras (install whichever you use):
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
pip install "fasticrl[openai]" # OpenAI
|
|
93
|
+
pip install "fasticrl[ollama]" # Ollama (local models)
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
Requires Python ≥ 3.13.
|
|
97
|
+
|
|
98
|
+
---
|
|
99
|
+
|
|
100
|
+
## Quick start
|
|
101
|
+
|
|
102
|
+
```python
|
|
103
|
+
from fasticrl import ICRLLearner
|
|
104
|
+
from agno.models.openai import OpenAIChat
|
|
105
|
+
|
|
106
|
+
model = OpenAIChat(id="gpt-4o-mini")
|
|
107
|
+
|
|
108
|
+
learner = ICRLLearner(
|
|
109
|
+
learner_model=model,
|
|
110
|
+
reward_model=model,
|
|
111
|
+
strategy_model=model,
|
|
112
|
+
task_description="Write a concise, compelling product description for an e-commerce listing.",
|
|
113
|
+
tasks=[
|
|
114
|
+
"Wireless noise-cancelling headphones",
|
|
115
|
+
"Ergonomic standing desk",
|
|
116
|
+
"Portable espresso maker",
|
|
117
|
+
],
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
# Run 3 episodes, update strategy every 2 steps, show progress bar
|
|
121
|
+
learner.auto_learn(episodes=3, batch_size=2, cli_mode=True, strategy_update_interval=2)
|
|
122
|
+
|
|
123
|
+
# Inspect what the agent learned
|
|
124
|
+
print(learner.strategy)
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
---
|
|
128
|
+
|
|
129
|
+
## API
|
|
130
|
+
|
|
131
|
+
### `ICRLLearner`
|
|
132
|
+
|
|
133
|
+
```python
|
|
134
|
+
ICRLLearner(
|
|
135
|
+
learner_model, # agno Model for the learner agent
|
|
136
|
+
reward_model, # agno Model for the reward agent
|
|
137
|
+
strategy_model, # agno Model for the strategist agent
|
|
138
|
+
task_description, # describes the overall task domain (required)
|
|
139
|
+
tasks, # list of concrete task instances to cycle through
|
|
140
|
+
buffer, # optional: pre-loaded list of Attempt objects
|
|
141
|
+
strategy, # optional: pre-loaded strategy string
|
|
142
|
+
)
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
#### Key methods
|
|
146
|
+
|
|
147
|
+
| Method | Description |
|
|
148
|
+
| --- | --- |
|
|
149
|
+
| `auto_learn(episodes, batch_size, cli_mode, strategy_update_interval)` | Run N episodes. `batch_size > 1` parallelizes tasks with a thread pool. `cli_mode=True` shows a progress bar. `strategy_update_interval=K` refreshes the strategy every K episodes. |
|
|
150
|
+
| `generate_action(task)` | Run the learner on a single task and return its output |
|
|
151
|
+
| `generate_reward(task, action)` | Score a learner output with the reward agent |
|
|
152
|
+
| `generate_attempt_by_present_task()` | Single step: generate + score the current task |
|
|
153
|
+
| `update_strategy()` | Ask the strategist to refine the strategy from the current buffer |
|
|
154
|
+
| `to_yaml(path)` | Persist the full agent state (buffer + strategy) to a YAML file |
|
|
155
|
+
| `ICRLLearner.from_yaml(path, ...)` | Resume from a saved state |
|
|
156
|
+
|
|
157
|
+
---
|
|
158
|
+
|
|
159
|
+
## Saving and resuming
|
|
160
|
+
|
|
161
|
+
```python
|
|
162
|
+
# Save
|
|
163
|
+
learner.to_yaml("my_agent.yaml")
|
|
164
|
+
|
|
165
|
+
# Resume later
|
|
166
|
+
learner = ICRLLearner.from_yaml(
|
|
167
|
+
"my_agent.yaml",
|
|
168
|
+
learner_model=model,
|
|
169
|
+
reward_model=model,
|
|
170
|
+
strategy_model=model,
|
|
171
|
+
)
|
|
172
|
+
learner.auto_learn(episodes=5)
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
---
|
|
176
|
+
|
|
177
|
+
## Using Ollama (local models)
|
|
178
|
+
|
|
179
|
+
```python
|
|
180
|
+
from agno.models.ollama import Ollama
|
|
181
|
+
|
|
182
|
+
learner = ICRLLearner(
|
|
183
|
+
learner_model=Ollama(id="llama3.2"),
|
|
184
|
+
reward_model=Ollama(id="llama3.2"),
|
|
185
|
+
strategy_model=Ollama(id="llama3.2"),
|
|
186
|
+
task_description="...",
|
|
187
|
+
tasks=[...],
|
|
188
|
+
)
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
Any [agno](https://github.com/agno-agi/agno)-compatible model works.
|
|
192
|
+
|
|
193
|
+
---
|
|
194
|
+
|
|
195
|
+
## Citation
|
|
196
|
+
|
|
197
|
+
This project is based on and inspired by the following papers:
|
|
198
|
+
|
|
199
|
+
*Reward Is Enough: LLMs Are In-Context Reinforcement Learners*
|
|
200
|
+
Kefan Song, Amir Moeini, Peng Wang, Lei Gong, Rohan Chandra, Shangtong Zhang, Yanjun Qi
|
|
201
|
+
arXiv:2506.06303 — [https://arxiv.org/abs/2506.06303](https://arxiv.org/abs/2506.06303)
|
|
202
|
+
|
|
203
|
+
*Large Language Models as Optimizers*
|
|
204
|
+
Chengrun Yang, Xuezhi Wang, Yifeng Lu, Hanxiao Liu, Quoc V. Le, Denny Zhou, Xinyun Chen
|
|
205
|
+
arXiv:2309.03409 — [https://arxiv.org/abs/2309.03409](https://arxiv.org/abs/2309.03409)
|
|
206
|
+
|
|
207
|
+
*Prompted Policy Search: Reinforcement Learning through Linguistic and Numerical Reasoning in LLMs*
|
|
208
|
+
Yifan Zhou, Sachin Grover, Mohamed El Mistiri, Kamalesh Kalirathinam, Pratyush Kerhalkar, Swaroop Mishra, Neelesh Kumar, Sanket Gaurav, Oya Aran, Heni Ben Amor
|
|
209
|
+
NeurIPS 2025 — [https://openreview.net/forum?id=95plu1Mo20](https://openreview.net/forum?id=95plu1Mo20)
|
|
210
|
+
|
|
211
|
+
---
|
|
212
|
+
|
|
213
|
+
## License
|
|
214
|
+
|
|
215
|
+
MIT
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
fasticrl/__init__.py,sha256=Z7wRa-FmjEfXbIfew6ZVzCLYsmw6ZVd-k2Zn4TCdPMU,128
|
|
2
|
+
fasticrl/icrl_learner.py,sha256=t-jvsox3Cs6BWthmZkwALByfwJcMH-6pyY0l5gG9hx0,6113
|
|
3
|
+
fasticrl/learner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
4
|
+
fasticrl/learner/core.py,sha256=1TvxqQUmZFKbdDCnkaL0njLPon9f2G2B3KpiZh8CzHs,975
|
|
5
|
+
fasticrl/learner/models/learn_output.py,sha256=4wjUrxMzm7KKIU7rFZK78y7BLwotvhHlxiJRZHaZulk,602
|
|
6
|
+
fasticrl/learner/prompts.py,sha256=_hECs059QOvgnnale6p2pwAhqW83th7ajb1k8vUzmzc,1573
|
|
7
|
+
fasticrl/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
|
+
fasticrl/models/agent_save_state.py,sha256=6kqG4v3yG3YX2RBLdi6Ja6cdtX7_mUMYFhg35Gmdeg0,308
|
|
9
|
+
fasticrl/models/attempt.py,sha256=xoLyKKFm3AlnB04QzbYqTZmIXNxfSu_vBYw4MKWUB7o,162
|
|
10
|
+
fasticrl/models/sentinel.py,sha256=7SPQ3mSqmlYZU9tTPDaDwObDArOmGYF3W642hja9nSU,21
|
|
11
|
+
fasticrl/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
12
|
+
fasticrl/reward/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
13
|
+
fasticrl/reward/core.py,sha256=Dnt7a7OWE_Ak9lssN5VNtdHvpTGgbgE2YYKg-MycHq0,724
|
|
14
|
+
fasticrl/reward/models/reward_output.py,sha256=LmoptYYmEk-5GJbhPRsNjfIwGt7VdRpki63IEU5Q2Ps,310
|
|
15
|
+
fasticrl/reward/prompts.py,sha256=F3RwDWNYWLtcIWwdREO019zMqY_cAWZrYyR3iE-74J4,1379
|
|
16
|
+
fasticrl/strategist/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
17
|
+
fasticrl/strategist/core.py,sha256=Aae-cX2FazxsYXrjeDWkMzR0nwQEcUYNJs2_CiaphgQ,944
|
|
18
|
+
fasticrl/strategist/models/strategy_output.py,sha256=LLQoyBc0qZ4iw2agEGXOOMKlCV8S46Yz7pW9nbW7y9w,242
|
|
19
|
+
fasticrl/strategist/prompts.py,sha256=ZoM-UXuRbQKIC3SECHzvUAhYGCnvLoqnRmg_YQgfJrU,1338
|
|
20
|
+
fasticrl-1.0.0.dist-info/WHEEL,sha256=eh7sammvW2TypMMMGKgsM83HyA_3qQ5Lgg3ynoecH3M,79
|
|
21
|
+
fasticrl-1.0.0.dist-info/METADATA,sha256=_nO4dFnI3qr2ecoklKpunTaFo_oNEC5fgQ-vl8VFr5g,7954
|
|
22
|
+
fasticrl-1.0.0.dist-info/RECORD,,
|