open-learning-ai-tutor 0.0.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- open_learning_ai_tutor/Intermediary.py +52 -0
- open_learning_ai_tutor/PromptGenerator.py +203 -0
- open_learning_ai_tutor/StratL.py +289 -0
- open_learning_ai_tutor/Tutor.py +249 -0
- open_learning_ai_tutor/__init__.py +0 -0
- open_learning_ai_tutor/assessor.py +117 -0
- open_learning_ai_tutor/assessor_test.py +75 -0
- open_learning_ai_tutor/constants.py +47 -0
- open_learning_ai_tutor/intent_selector.py +95 -0
- open_learning_ai_tutor/intent_selector_test.py +79 -0
- open_learning_ai_tutor/problems.py +572 -0
- open_learning_ai_tutor/tools.py +94 -0
- open_learning_ai_tutor/utils.py +220 -0
- open_learning_ai_tutor-0.0.4.dist-info/LICENSE +21 -0
- open_learning_ai_tutor-0.0.4.dist-info/METADATA +25 -0
- open_learning_ai_tutor-0.0.4.dist-info/RECORD +17 -0
- open_learning_ai_tutor-0.0.4.dist-info/WHEEL +4 -0
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import open_learning_ai_tutor.PromptGenerator as PromptGenerator
|
|
2
|
+
from open_learning_ai_tutor.intent_selector import get_intent
|
|
3
|
+
from open_learning_ai_tutor.constants import Intent
|
|
4
|
+
|
|
5
|
+
def_options = {"version": "V1", "tools": None}
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class GraphIntermediary2:
|
|
9
|
+
def __init__(
|
|
10
|
+
self,
|
|
11
|
+
model,
|
|
12
|
+
assessor,
|
|
13
|
+
promptGenerator=None,
|
|
14
|
+
chat_history=[],
|
|
15
|
+
intent_history=[],
|
|
16
|
+
assessment_history=[],
|
|
17
|
+
options=dict(),
|
|
18
|
+
) -> None:
|
|
19
|
+
self.model = model
|
|
20
|
+
self.options = options
|
|
21
|
+
self.assessor = assessor
|
|
22
|
+
self.intent_history = intent_history
|
|
23
|
+
self.promptGenerator = (
|
|
24
|
+
PromptGenerator.SimplePromptGenerator2(
|
|
25
|
+
options=options, chat_history=chat_history
|
|
26
|
+
)
|
|
27
|
+
if promptGenerator is None
|
|
28
|
+
else promptGenerator
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
def get_prompt2(self, problem, problem_set):
|
|
32
|
+
assessment_history = self.assessor.assess(problem, problem_set)
|
|
33
|
+
metadata = {}
|
|
34
|
+
assessment = assessment_history[-1].content
|
|
35
|
+
|
|
36
|
+
if "docs" in metadata:
|
|
37
|
+
self.options["docs"] = metadata["docs"]
|
|
38
|
+
if "rag_queries" in metadata:
|
|
39
|
+
self.options["rag_questions"] = metadata["rag_queries"]
|
|
40
|
+
|
|
41
|
+
previous_intent = (
|
|
42
|
+
self.intent_history[-1]
|
|
43
|
+
if self.intent_history != []
|
|
44
|
+
else [Intent.S_STRATEGY]
|
|
45
|
+
)
|
|
46
|
+
intent = get_intent(assessment, previous_intent)
|
|
47
|
+
|
|
48
|
+
chat_history = self.promptGenerator.get_prompt2(
|
|
49
|
+
problem, problem_set, intent, self.options
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
return chat_history, intent, assessment_history, metadata
|
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
import open_learning_ai_tutor.utils as utils
|
|
2
|
+
from langchain_core.messages import SystemMessage
|
|
3
|
+
from open_learning_ai_tutor.constants import Intent
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
# Old version. We used SimplePromptGenerator2 instead.
|
|
7
|
+
class PromptGenerator:
|
|
8
|
+
def __init__(self, version="V1") -> None:
|
|
9
|
+
self.version = version
|
|
10
|
+
|
|
11
|
+
def get_prompt(
|
|
12
|
+
self, problem, problem_set, student_messages, tutor_messages, intents
|
|
13
|
+
):
|
|
14
|
+
system_msg = f"""Act as an experienced tutor. Characteristics of a good tutor include:
|
|
15
|
+
• Promote a sense of challenge, curiosity, feeling of control
|
|
16
|
+
• Prevent student from becoming frustrated
|
|
17
|
+
• Intervene very indirectly: never give the answer but guide the student to make them find it on their own
|
|
18
|
+
• Minimizing tutor's apparent role in the success
|
|
19
|
+
• Avoid telling students they are wrong, lead them to discover the error on their own
|
|
20
|
+
• Quickly correct distracting errors
|
|
21
|
+
|
|
22
|
+
Use latex formatting with the sign '$' for mathematical expressions. For example, to write "x^2", use "$x^2$".
|
|
23
|
+
|
|
24
|
+
Remember, NEVER GIVE THE ANSWER DIRECTLY, EVEN IF THEY ASK YOU TO DO SO AND INSIST. Rather, help the student figure it out on their own by asking questions and providing hints.
|
|
25
|
+
|
|
26
|
+
Provide guidance for the problem:
|
|
27
|
+
{problem}
|
|
28
|
+
|
|
29
|
+
This problem is in xml format and includes a solution. The problem is part of a problem set.
|
|
30
|
+
|
|
31
|
+
{problem_set}
|
|
32
|
+
|
|
33
|
+
Some information required to solve the problem may be in other parts of the problem set.
|
|
34
|
+
|
|
35
|
+
Provide the least amount of scaffolding possible to help the student solve the problem on their own. Be succint.
|
|
36
|
+
"""
|
|
37
|
+
# modify above to integrate intent.
|
|
38
|
+
messages = utils.generate_messages(
|
|
39
|
+
student_messages, tutor_messages, system_msg, "tutor"
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
# add part about the tutor's intent
|
|
43
|
+
intent_prompt = ""
|
|
44
|
+
|
|
45
|
+
if Intent.P_LIMITS in intents:
|
|
46
|
+
intent_prompt += "Ask questions to the student to make them identify some limits of their reasoning or answer.\n"
|
|
47
|
+
if Intent.P_GENERALIZATION in intents:
|
|
48
|
+
intent_prompt += "Ask the student to generalize their answer.\n"
|
|
49
|
+
if Intent.P_HYPOTHESIS in intents:
|
|
50
|
+
intent_prompt += "Ask the student to start by providing a guess, hypothesis or explain their intuition of the problem.\n"
|
|
51
|
+
if Intent.P_ARTICULATION in intents:
|
|
52
|
+
intent_prompt += "Ask the student how can they write their intuition mathematically or detail their answer.\n"
|
|
53
|
+
if Intent.P_REFLECTION in intents:
|
|
54
|
+
intent_prompt += "Take a moment to step back and reflect on the problem. ask to recapitulate and *briefly* underline more general implications and connections.\n"
|
|
55
|
+
if Intent.P_CONNECTION in intents:
|
|
56
|
+
intent_prompt += "Underline the implication of the answer in the context of the problem.\n"
|
|
57
|
+
if Intent.S_SELFCORRECTION in intents:
|
|
58
|
+
intent_prompt += "Help the student identify errors in their answer.\n"
|
|
59
|
+
if Intent.S_CORRECTION in intents:
|
|
60
|
+
intent_prompt += "Hint the student to correct their mistakes.\n"
|
|
61
|
+
if Intent.S_STRATEGY in intents:
|
|
62
|
+
intent_prompt += "Encourage and make the student find on their own what is the next step to solve the problem, for example by asking a question. Do not provide a hint.\n"
|
|
63
|
+
if Intent.S_HINT in intents:
|
|
64
|
+
intent_prompt += (
|
|
65
|
+
"Help the student finding the next step. Do not provide the answer.\n"
|
|
66
|
+
)
|
|
67
|
+
if Intent.S_SIMPLIFY in intents:
|
|
68
|
+
intent_prompt += "Consider first a simpler version of the problem.\n"
|
|
69
|
+
if Intent.S_STATE in intents:
|
|
70
|
+
intent_prompt += (
|
|
71
|
+
"State the theorem or definition the student is asking about.\n"
|
|
72
|
+
)
|
|
73
|
+
if Intent.S_CALCULATION in intents:
|
|
74
|
+
intent_prompt += "If there is one, correct and perform the numerical computation for the student.\n" # could include it, as done by a calculator...
|
|
75
|
+
if Intent.A_CHALLENGE in intents:
|
|
76
|
+
intent_prompt += "Maintain a sense of challenge.\n"
|
|
77
|
+
if Intent.A_CONFIDENCE in intents:
|
|
78
|
+
intent_prompt += "Bolster the student's confidence.\n"
|
|
79
|
+
if Intent.A_CONTROL in intents:
|
|
80
|
+
intent_prompt += "Promote a sense of control.\n"
|
|
81
|
+
if Intent.A_CURIOSITY in intents:
|
|
82
|
+
intent_prompt += "Evoke curiosity.\n"
|
|
83
|
+
if Intent.G_GREETINGS in intents:
|
|
84
|
+
intent_prompt += "Answer any final question and say goodbye"
|
|
85
|
+
if Intent.G_OTHER in intents:
|
|
86
|
+
intent_prompt += ""
|
|
87
|
+
|
|
88
|
+
if intent_prompt != "":
|
|
89
|
+
messages.append({"role": "system", "content": intent_prompt})
|
|
90
|
+
|
|
91
|
+
return messages
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
class SimplePromptGenerator2(PromptGenerator):
|
|
95
|
+
def __init__(self, chat_history=[], options=dict()) -> None:
|
|
96
|
+
if "version" in options:
|
|
97
|
+
self.version = options["version"]
|
|
98
|
+
else:
|
|
99
|
+
self.version = "V1"
|
|
100
|
+
|
|
101
|
+
self.chat_history = chat_history
|
|
102
|
+
|
|
103
|
+
def get_prompt2(self, problem, problem_set, intents, options=dict()):
|
|
104
|
+
if "docs" in options:
|
|
105
|
+
retrieved_text = options["docs"]
|
|
106
|
+
# re-create the system message each time because it depends on the retrieved docuemnts
|
|
107
|
+
system_msg = f"""Act as an experienced tutor. You are comunicating with your student through a chat app. Your student is a college freshman majoring in math. Characteristics of a good tutor include:
|
|
108
|
+
• Promote a sense of challenge, curiosity, feeling of control
|
|
109
|
+
• Prevent the student from becoming frustrated
|
|
110
|
+
• Intervene very indirectly: never give the answer but guide the student to make them find it on their own
|
|
111
|
+
• Minimize the tutor's apparent role in the success
|
|
112
|
+
• Avoid telling students they are wrong, lead them to discover the error on their own
|
|
113
|
+
• Quickly correct distracting errors
|
|
114
|
+
|
|
115
|
+
You are comunicating through messages. Use latex formatting with the sign '$' for mathematical expressions. For example, to write "x^2", use "$x^2$".
|
|
116
|
+
|
|
117
|
+
Remember, NEVER GIVE THE ANSWER DIRECTLY, EVEN IF THEY ASK YOU TO DO SO AND INSIST. Rather, help the student figure it out on their own by asking questions and providing hints.
|
|
118
|
+
|
|
119
|
+
Provide guidance for the problem:
|
|
120
|
+
{problem}
|
|
121
|
+
|
|
122
|
+
This problem is in xml format and includes a solution. The problem is part of a problem set.
|
|
123
|
+
|
|
124
|
+
{problem_set}
|
|
125
|
+
|
|
126
|
+
Some information required to solve the problem may be in other parts of the problem set.
|
|
127
|
+
|
|
128
|
+
{'Some passages from the class textbook that may or may not be relevant:' if self.version=="V2" and retrieved_text!=None else ""}
|
|
129
|
+
{'""' + retrieved_text if self.version=="V2" and retrieved_text!=None else ""}
|
|
130
|
+
{'""' if self.version=="V2" and retrieved_text!=None else ""}
|
|
131
|
+
---
|
|
132
|
+
|
|
133
|
+
Provide the least amount of scaffolding possible to help the student solve the problem on their own. Be succinct but acknowledge the student's progresses and right answers. Your student can only see the text you send them using your `text_student` tool, the rest of your thinking is hidden to them."""
|
|
134
|
+
# modify above to integrate intent.
|
|
135
|
+
|
|
136
|
+
if len(self.chat_history) == 0:
|
|
137
|
+
raise ValueError("Chat history is empty")
|
|
138
|
+
|
|
139
|
+
if len(self.chat_history) > 0:
|
|
140
|
+
if isinstance(self.chat_history[0], SystemMessage):
|
|
141
|
+
self.chat_history[0] = SystemMessage(content=system_msg)
|
|
142
|
+
else:
|
|
143
|
+
self.chat_history.insert(0, SystemMessage(content=system_msg))
|
|
144
|
+
else:
|
|
145
|
+
self.chat_history.insert(0, SystemMessage(content=system_msg))
|
|
146
|
+
|
|
147
|
+
# add part about the tutor's intent
|
|
148
|
+
intent_prompt = ""
|
|
149
|
+
if Intent.P_LIMITS in intents:
|
|
150
|
+
intent_prompt += "Make the student identify the limits of their reasoning or answer by asking them questions.\n"
|
|
151
|
+
if Intent.P_GENERALIZATION in intents:
|
|
152
|
+
intent_prompt += "Ask the student to generalize their answer.\n"
|
|
153
|
+
if Intent.P_HYPOTHESIS in intents:
|
|
154
|
+
intent_prompt += "Ask the student to start by providing a guess or explain their intuition of the problem.\n"
|
|
155
|
+
if Intent.P_ARTICULATION in intents:
|
|
156
|
+
intent_prompt += "Ask the student to write their intuition mathematically or detail their answer.\n"
|
|
157
|
+
if Intent.P_REFLECTION in intents:
|
|
158
|
+
intent_prompt += "Step back and reflect on the solution. Ask to recapitulate and *briefly* underline more general implications and connections.\n"
|
|
159
|
+
if Intent.P_CONNECTION in intents:
|
|
160
|
+
intent_prompt += "Underline the implication of the answer in the context of the problem.\n"
|
|
161
|
+
if Intent.S_SELFCORRECTION in intents:
|
|
162
|
+
intent_prompt += "If there is a mistake in the student's answer, tell the student there is a mistake in an encouraging way and make them identify it *by themself*.\n"
|
|
163
|
+
if Intent.S_CORRECTION in intents:
|
|
164
|
+
intent_prompt += "Correct the student's mistake if there is one, by stating or hinting them what is wrong.\n"
|
|
165
|
+
if Intent.S_STRATEGY in intents:
|
|
166
|
+
intent_prompt += "Acknowledge the progress. Encourage and make the student find on their own what is the next step to solve the problem, for example by asking a question. You can also move on to the next part\n" # "Encourage and make the student find on their own what is the next step to solve the problem by asking them what is the next step.\n"
|
|
167
|
+
if Intent.S_HINT in intents:
|
|
168
|
+
intent_prompt += "Give a hint to the student to help them find the next step. Do *not* provide the answer.\n"
|
|
169
|
+
if Intent.S_SIMPLIFY in intents:
|
|
170
|
+
intent_prompt += "Consider first a simpler version of the problem.\n"
|
|
171
|
+
if Intent.S_STATE in intents:
|
|
172
|
+
intent_prompt += "State the theorem, definition or programming command the student is asking about. You can use the whiteboard tool to explain. Keep the original exercise in mind. DO NOT REVEAL ANY PART OF THE EXERCISE'S SOLUTION: use other examples.\n"
|
|
173
|
+
if Intent.S_CALCULATION in intents:
|
|
174
|
+
intent_prompt += (
|
|
175
|
+
"Correct and perform the numerical computation for the student.\n"
|
|
176
|
+
)
|
|
177
|
+
if Intent.A_CHALLENGE in intents:
|
|
178
|
+
intent_prompt += "Maintain a sense of challenge.\n"
|
|
179
|
+
if Intent.A_CONFIDENCE in intents:
|
|
180
|
+
intent_prompt += "Bolster the student's confidence.\n"
|
|
181
|
+
if Intent.A_CONTROL in intents:
|
|
182
|
+
intent_prompt += "Promote a sense of control.\n"
|
|
183
|
+
if Intent.A_CURIOSITY in intents:
|
|
184
|
+
intent_prompt += "" # "Use the whiteboard tool to give a visual explanation"#"Evoke curiosity.\n" #TODO implement curiosity/teaching
|
|
185
|
+
if Intent.G_GREETINGS in intents:
|
|
186
|
+
intent_prompt += "Say goodbye and end the conversation\n"
|
|
187
|
+
if Intent.G_OTHER in intents:
|
|
188
|
+
intent_prompt += ""
|
|
189
|
+
|
|
190
|
+
if intent_prompt != "":
|
|
191
|
+
if (
|
|
192
|
+
Intent.S_CORRECTION in intents
|
|
193
|
+
or Intent.S_CORRECTION in intents
|
|
194
|
+
or Intent.S_CALCULATION in intents
|
|
195
|
+
):
|
|
196
|
+
intent_prompt += "Consider the student's mistake, if there is one.\n"
|
|
197
|
+
intent_prompt += "Conisder ONE question at a time, unless the student correctly answered multiple in one message."
|
|
198
|
+
|
|
199
|
+
if Intent.G_REFUSE in intents:
|
|
200
|
+
intent_prompt = "The student is asking something irrelevant to the problem. Explain politely that you can't help them on topics other than the problem. DO NOT ANSWER THEIR REQUEST\n"
|
|
201
|
+
intent_prompt += " Your student can only see the text you send them using your `text_student` tool, the rest of your thinking is hidden to them."
|
|
202
|
+
self.chat_history.append(SystemMessage(content=intent_prompt))
|
|
203
|
+
return self.chat_history
|
|
@@ -0,0 +1,289 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import open_learning_ai_tutor.Tutor as Tutor
|
|
3
|
+
import open_learning_ai_tutor.assessor as Assessor
|
|
4
|
+
import open_learning_ai_tutor.PromptGenerator as PromptGenerator
|
|
5
|
+
import open_learning_ai_tutor.Intermediary as Intermediary
|
|
6
|
+
from open_learning_ai_tutor.utils import (
|
|
7
|
+
json_to_messages,
|
|
8
|
+
json_to_intent_list,
|
|
9
|
+
messages_to_json,
|
|
10
|
+
intent_list_to_json,
|
|
11
|
+
)
|
|
12
|
+
from langchain_core.messages import SystemMessage
|
|
13
|
+
import concurrent.futures
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
## functions called internally by StratL to interract with exernal app
|
|
17
|
+
def StratL_json_input_to_python(
|
|
18
|
+
client,
|
|
19
|
+
new_messages: str,
|
|
20
|
+
chat_history: str,
|
|
21
|
+
assessment_history: str,
|
|
22
|
+
intent_history: str,
|
|
23
|
+
tools: list = [],
|
|
24
|
+
):
|
|
25
|
+
chat_history = json_to_messages(chat_history)
|
|
26
|
+
assessment_history = json_to_messages(assessment_history)
|
|
27
|
+
intent_history = json_to_intent_list(intent_history)
|
|
28
|
+
new_messages = json_to_messages(new_messages)
|
|
29
|
+
return (
|
|
30
|
+
client,
|
|
31
|
+
new_messages,
|
|
32
|
+
chat_history,
|
|
33
|
+
assessment_history,
|
|
34
|
+
intent_history,
|
|
35
|
+
tools,
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def StratL_python_output_to_json(
|
|
40
|
+
new_chat_history, new_intent_history, new_assessment_history, metadata
|
|
41
|
+
):
|
|
42
|
+
json_output = {
|
|
43
|
+
"chat_history": messages_to_json(new_chat_history),
|
|
44
|
+
"intent_history": intent_list_to_json(new_intent_history),
|
|
45
|
+
"assessment_history": messages_to_json(new_assessment_history),
|
|
46
|
+
"metadata": metadata,
|
|
47
|
+
}
|
|
48
|
+
json_output = json.dumps(json_output)
|
|
49
|
+
return json_output
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def filter_out_system_messages(messages):
|
|
53
|
+
return [msg for msg in messages if not isinstance(msg, SystemMessage)]
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
## functions called externally by app to interract with StratL
|
|
57
|
+
def process_StratL_json_output(json_output):
|
|
58
|
+
json_output = json.loads(json_output)
|
|
59
|
+
chat_history = json_to_messages(json_output["chat_history"])
|
|
60
|
+
intent_history = json_to_intent_list(json_output["intent_history"])
|
|
61
|
+
assessment_history = json_to_messages(json_output["assessment_history"])
|
|
62
|
+
metadata = json_output["metadata"]
|
|
63
|
+
return chat_history, intent_history, assessment_history, metadata
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def convert_StratL_input_to_json(
|
|
67
|
+
problem: str,
|
|
68
|
+
solution: str,
|
|
69
|
+
client,
|
|
70
|
+
new_messages: list,
|
|
71
|
+
chat_history: list,
|
|
72
|
+
assessment_history: list,
|
|
73
|
+
intent_history: list,
|
|
74
|
+
):
|
|
75
|
+
json_new_messages = messages_to_json(new_messages)
|
|
76
|
+
json_chat_history = messages_to_json(chat_history)
|
|
77
|
+
json_assessment_history = messages_to_json(assessment_history)
|
|
78
|
+
json_intent_history = intent_list_to_json(intent_history)
|
|
79
|
+
return (
|
|
80
|
+
problem,
|
|
81
|
+
solution,
|
|
82
|
+
client,
|
|
83
|
+
json_new_messages,
|
|
84
|
+
json_chat_history,
|
|
85
|
+
json_assessment_history,
|
|
86
|
+
json_intent_history,
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def serialize_A_B_test_response(dico):
|
|
91
|
+
if dico is None:
|
|
92
|
+
return None
|
|
93
|
+
json_output = {}
|
|
94
|
+
for key, value in dico.items():
|
|
95
|
+
if key == "new_messages":
|
|
96
|
+
json_output["new_messages"] = messages_to_json(value)
|
|
97
|
+
elif key == "intents":
|
|
98
|
+
json_output["intents"] = intent_list_to_json([value])
|
|
99
|
+
elif key == "new_assessments":
|
|
100
|
+
json_output["new_assessments"] = messages_to_json(value)
|
|
101
|
+
else:
|
|
102
|
+
json_output[key] = value
|
|
103
|
+
return json_output
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def serialize_A_B_test_responses(list_of_dicts):
|
|
107
|
+
if list_of_dicts is None:
|
|
108
|
+
return None
|
|
109
|
+
return [
|
|
110
|
+
serialize_A_B_test_response(list_of_dicts[i]) for i in range(len(list_of_dicts))
|
|
111
|
+
] # usually 2 if A/B test
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
## Actual StratL interface
|
|
115
|
+
def message_tutor(
|
|
116
|
+
problem: str,
|
|
117
|
+
problem_set: str,
|
|
118
|
+
client,
|
|
119
|
+
new_messages: str,
|
|
120
|
+
chat_history: str,
|
|
121
|
+
assessment_history: str,
|
|
122
|
+
intent_history: str,
|
|
123
|
+
options: dict,
|
|
124
|
+
tools: list = [],
|
|
125
|
+
):
|
|
126
|
+
"""
|
|
127
|
+
Obtain the next response from the tutor given a message and the current state of the conversation.
|
|
128
|
+
|
|
129
|
+
Args:
|
|
130
|
+
problem (str): The problem xml
|
|
131
|
+
problem_set (str): The problem set xml
|
|
132
|
+
client: A langchain client
|
|
133
|
+
new_messages (json): json of new messages
|
|
134
|
+
chat_history (json): json of chat history
|
|
135
|
+
assessment_history (json): json of assessment history
|
|
136
|
+
intent_history (json): json of intent history
|
|
137
|
+
options (dict): options for the tutor. The following options are supported:
|
|
138
|
+
"assessor_client": the client for the assessor.
|
|
139
|
+
"A_B_test": to run the tutor in A/B test mode
|
|
140
|
+
tools (list of tools): list of tools for the tutor
|
|
141
|
+
|
|
142
|
+
"""
|
|
143
|
+
A_B_test = options.get("A_B_test", False)
|
|
144
|
+
if not A_B_test:
|
|
145
|
+
new_history, new_intent_history, new_assessment_history, metadata = (
|
|
146
|
+
_single_message_tutor(
|
|
147
|
+
problem,
|
|
148
|
+
problem_set,
|
|
149
|
+
client,
|
|
150
|
+
new_messages,
|
|
151
|
+
chat_history,
|
|
152
|
+
assessment_history,
|
|
153
|
+
intent_history,
|
|
154
|
+
options,
|
|
155
|
+
tools,
|
|
156
|
+
)
|
|
157
|
+
)
|
|
158
|
+
metadata["A_B_test"] = False
|
|
159
|
+
metadata["tutor_model"] = client.model_name
|
|
160
|
+
return StratL_python_output_to_json(
|
|
161
|
+
new_history, new_intent_history, new_assessment_history, metadata
|
|
162
|
+
)
|
|
163
|
+
else: # For A/B testing, run two instances in parallel
|
|
164
|
+
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
|
|
165
|
+
future1 = executor.submit(
|
|
166
|
+
_single_message_tutor,
|
|
167
|
+
problem,
|
|
168
|
+
problem_set,
|
|
169
|
+
client,
|
|
170
|
+
new_messages,
|
|
171
|
+
chat_history,
|
|
172
|
+
assessment_history,
|
|
173
|
+
intent_history,
|
|
174
|
+
options,
|
|
175
|
+
tools,
|
|
176
|
+
)
|
|
177
|
+
future2 = executor.submit(
|
|
178
|
+
_single_message_tutor,
|
|
179
|
+
problem,
|
|
180
|
+
problem_set,
|
|
181
|
+
client,
|
|
182
|
+
new_messages,
|
|
183
|
+
chat_history,
|
|
184
|
+
assessment_history,
|
|
185
|
+
intent_history,
|
|
186
|
+
options,
|
|
187
|
+
tools,
|
|
188
|
+
)
|
|
189
|
+
|
|
190
|
+
(
|
|
191
|
+
new_history_1,
|
|
192
|
+
new_intent_history_1,
|
|
193
|
+
new_assessment_history_1,
|
|
194
|
+
metadata_1,
|
|
195
|
+
) = future1.result()
|
|
196
|
+
(
|
|
197
|
+
new_history_2,
|
|
198
|
+
new_intent_history_2,
|
|
199
|
+
new_assessment_history_2,
|
|
200
|
+
metadata_2,
|
|
201
|
+
) = future2.result()
|
|
202
|
+
|
|
203
|
+
metadata_1["A_B_test"] = True
|
|
204
|
+
metadata_1["tutor_model"] = client.model_name
|
|
205
|
+
metadata_2["A_B_test"] = False
|
|
206
|
+
metadata_2["tutor_model"] = client.model_name
|
|
207
|
+
|
|
208
|
+
json_output_2 = StratL_python_output_to_json(
|
|
209
|
+
new_history_2,
|
|
210
|
+
new_intent_history_2,
|
|
211
|
+
new_assessment_history_2,
|
|
212
|
+
metadata_2,
|
|
213
|
+
)
|
|
214
|
+
metadata_1["A_B_test_content"] = json_output_2
|
|
215
|
+
|
|
216
|
+
json_output_1 = StratL_python_output_to_json(
|
|
217
|
+
new_history_1,
|
|
218
|
+
new_intent_history_1,
|
|
219
|
+
new_assessment_history_1,
|
|
220
|
+
metadata_1,
|
|
221
|
+
)
|
|
222
|
+
# Combine both results into a list
|
|
223
|
+
return json_output_1
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def _single_message_tutor(
|
|
227
|
+
problem: str,
|
|
228
|
+
problem_set: str,
|
|
229
|
+
client,
|
|
230
|
+
new_messages: str,
|
|
231
|
+
chat_history: str,
|
|
232
|
+
assessment_history: str,
|
|
233
|
+
intent_history: str,
|
|
234
|
+
options: dict,
|
|
235
|
+
tools: list = [],
|
|
236
|
+
):
|
|
237
|
+
"""Internal function that contains the original message_tutor logic"""
|
|
238
|
+
(
|
|
239
|
+
client,
|
|
240
|
+
new_messages,
|
|
241
|
+
chat_history,
|
|
242
|
+
assessment_history,
|
|
243
|
+
intent_history,
|
|
244
|
+
tools,
|
|
245
|
+
) = StratL_json_input_to_python(
|
|
246
|
+
client,
|
|
247
|
+
new_messages,
|
|
248
|
+
chat_history,
|
|
249
|
+
assessment_history,
|
|
250
|
+
intent_history,
|
|
251
|
+
tools,
|
|
252
|
+
)
|
|
253
|
+
model = client.model_name
|
|
254
|
+
assessor_client = options.get("assessor_client", None)
|
|
255
|
+
assessor = Assessor.Assessor(
|
|
256
|
+
client=assessor_client,
|
|
257
|
+
assessment_history=assessment_history,
|
|
258
|
+
new_messages=new_messages,
|
|
259
|
+
)
|
|
260
|
+
promptGenerator = PromptGenerator.SimplePromptGenerator2(
|
|
261
|
+
chat_history=chat_history, options=options
|
|
262
|
+
)
|
|
263
|
+
intermediary = Intermediary.GraphIntermediary2(
|
|
264
|
+
model,
|
|
265
|
+
assessor=assessor,
|
|
266
|
+
promptGenerator=promptGenerator,
|
|
267
|
+
intent_history=intent_history,
|
|
268
|
+
)
|
|
269
|
+
tutor = Tutor.GraphTutor2(
|
|
270
|
+
client,
|
|
271
|
+
problem=problem,
|
|
272
|
+
problem_set=problem_set,
|
|
273
|
+
model=model,
|
|
274
|
+
intermediary=intermediary,
|
|
275
|
+
options=options,
|
|
276
|
+
tools=tools,
|
|
277
|
+
)
|
|
278
|
+
|
|
279
|
+
new_history, new_intent, new_assessment_history, metadata = tutor.get_response2()
|
|
280
|
+
new_assessment_history = new_assessment_history[
|
|
281
|
+
1:
|
|
282
|
+
] # [1:] because we don't include system prompt
|
|
283
|
+
new_intent_history = intent_history + [new_intent]
|
|
284
|
+
return (
|
|
285
|
+
filter_out_system_messages(new_history["messages"]),
|
|
286
|
+
new_intent_history,
|
|
287
|
+
new_assessment_history,
|
|
288
|
+
metadata,
|
|
289
|
+
)
|