fabricatio 0.2.7.dev4__cp312-cp312-win_amd64.whl → 0.2.8__cp312-cp312-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fabricatio/__init__.py +4 -11
- fabricatio/actions/article.py +226 -92
- fabricatio/actions/article_rag.py +86 -21
- fabricatio/actions/output.py +71 -3
- fabricatio/actions/rag.py +3 -3
- fabricatio/actions/rules.py +39 -0
- fabricatio/capabilities/advanced_judge.py +23 -0
- fabricatio/capabilities/censor.py +90 -0
- fabricatio/capabilities/check.py +195 -0
- fabricatio/capabilities/correct.py +160 -96
- fabricatio/capabilities/propose.py +20 -4
- fabricatio/capabilities/rag.py +5 -4
- fabricatio/capabilities/rating.py +68 -23
- fabricatio/capabilities/review.py +21 -190
- fabricatio/capabilities/task.py +9 -10
- fabricatio/config.py +11 -3
- fabricatio/fs/curd.py +4 -0
- fabricatio/models/action.py +24 -10
- fabricatio/models/adv_kwargs_types.py +25 -0
- fabricatio/models/extra/__init__.py +1 -0
- fabricatio/models/extra/advanced_judge.py +32 -0
- fabricatio/models/extra/article_base.py +324 -89
- fabricatio/models/extra/article_essence.py +49 -176
- fabricatio/models/extra/article_main.py +48 -127
- fabricatio/models/extra/article_outline.py +12 -152
- fabricatio/models/extra/article_proposal.py +29 -13
- fabricatio/models/extra/patches.py +7 -0
- fabricatio/models/extra/problem.py +153 -0
- fabricatio/models/extra/rule.py +65 -0
- fabricatio/models/generic.py +360 -88
- fabricatio/models/kwargs_types.py +23 -17
- fabricatio/models/role.py +4 -1
- fabricatio/models/task.py +1 -1
- fabricatio/models/tool.py +149 -14
- fabricatio/models/usages.py +61 -47
- fabricatio/models/utils.py +0 -46
- fabricatio/parser.py +7 -8
- fabricatio/rust.cp312-win_amd64.pyd +0 -0
- fabricatio/{_rust.pyi → rust.pyi} +50 -0
- fabricatio/{_rust_instances.py → rust_instances.py} +1 -1
- fabricatio/utils.py +54 -0
- fabricatio-0.2.8.data/scripts/tdown.exe +0 -0
- {fabricatio-0.2.7.dev4.dist-info → fabricatio-0.2.8.dist-info}/METADATA +2 -1
- fabricatio-0.2.8.dist-info/RECORD +58 -0
- fabricatio/_rust.cp312-win_amd64.pyd +0 -0
- fabricatio-0.2.7.dev4.data/scripts/tdown.exe +0 -0
- fabricatio-0.2.7.dev4.dist-info/RECORD +0 -47
- {fabricatio-0.2.7.dev4.dist-info → fabricatio-0.2.8.dist-info}/WHEEL +0 -0
- {fabricatio-0.2.7.dev4.dist-info → fabricatio-0.2.8.dist-info}/licenses/LICENSE +0 -0
@@ -1,145 +1,209 @@
|
|
1
|
-
"""Correct capability
|
1
|
+
"""A module containing the Correct capability for reviewing, validating, and improving objects."""
|
2
2
|
|
3
|
-
|
4
|
-
to provide mechanisms for reviewing, validating, and correcting various objects and tasks
|
5
|
-
based on predefined criteria and templates.
|
6
|
-
"""
|
3
|
+
from typing import Optional, Type, Unpack, cast
|
7
4
|
|
8
|
-
from
|
9
|
-
|
10
|
-
from fabricatio._rust_instances import TEMPLATE_MANAGER
|
11
|
-
from fabricatio.capabilities.review import Review, ReviewResult
|
5
|
+
from fabricatio.capabilities.propose import Propose
|
6
|
+
from fabricatio.capabilities.rating import Rating
|
12
7
|
from fabricatio.config import configs
|
13
|
-
from fabricatio.
|
14
|
-
from fabricatio.models.
|
15
|
-
from fabricatio.models.
|
16
|
-
from
|
17
|
-
from
|
8
|
+
from fabricatio.journal import logger
|
9
|
+
from fabricatio.models.adv_kwargs_types import CorrectKwargs
|
10
|
+
from fabricatio.models.extra.problem import Improvement, ProblemSolutions
|
11
|
+
from fabricatio.models.generic import ProposedUpdateAble, SketchedAble
|
12
|
+
from fabricatio.models.kwargs_types import (
|
13
|
+
BestKwargs,
|
14
|
+
ValidateKwargs,
|
15
|
+
)
|
16
|
+
from fabricatio.rust_instances import TEMPLATE_MANAGER
|
17
|
+
from fabricatio.utils import ok, override_kwargs
|
18
|
+
|
19
|
+
|
20
|
+
class Correct(Rating, Propose):
|
21
|
+
"""A class that provides the capability to correct objects."""
|
22
|
+
|
23
|
+
async def decide_solution(
|
24
|
+
self, problem_solutions: ProblemSolutions, **kwargs: Unpack[BestKwargs]
|
25
|
+
) -> ProblemSolutions:
|
26
|
+
"""Decide the best solution from a list of problem solutions.
|
27
|
+
|
28
|
+
Args:
|
29
|
+
problem_solutions (ProblemSolutions): The problem solutions to evaluate.
|
30
|
+
**kwargs (Unpack[BestKwargs]): Additional keyword arguments for the decision process.
|
18
31
|
|
32
|
+
Returns:
|
33
|
+
ProblemSolutions: The problem solutions with the best solution selected.
|
34
|
+
"""
|
35
|
+
if (leng := len(problem_solutions.solutions)) == 0:
|
36
|
+
logger.error(f"No solutions found in ProblemSolutions, Skip: {problem_solutions.problem}")
|
37
|
+
if leng > 1:
|
38
|
+
problem_solutions.solutions = await self.best(problem_solutions.solutions, **kwargs)
|
39
|
+
return problem_solutions
|
19
40
|
|
20
|
-
|
21
|
-
|
41
|
+
async def decide_improvement(self, improvement: Improvement, **kwargs: Unpack[BestKwargs]) -> Improvement:
|
42
|
+
"""Decide the best solution for each problem solution in an improvement.
|
22
43
|
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
the required interfaces, applying corrections based on templated review processes.
|
27
|
-
"""
|
44
|
+
Args:
|
45
|
+
improvement (Improvement): The improvement containing problem solutions to evaluate.
|
46
|
+
**kwargs (Unpack[BestKwargs]): Additional keyword arguments for the decision process.
|
28
47
|
|
29
|
-
|
48
|
+
Returns:
|
49
|
+
Improvement: The improvement with the best solutions selected for each problem solution.
|
50
|
+
"""
|
51
|
+
if (leng := len(improvement.problem_solutions)) == 0:
|
52
|
+
logger.error(f"No problem_solutions found in Improvement, Skip: {improvement}")
|
53
|
+
if leng > 1:
|
54
|
+
for ps in improvement.problem_solutions:
|
55
|
+
ps.solutions = await self.best(ps.solutions, **kwargs)
|
56
|
+
return improvement
|
57
|
+
|
58
|
+
async def fix_troubled_obj[M: SketchedAble](
|
30
59
|
self,
|
31
60
|
obj: M,
|
61
|
+
problem_solutions: ProblemSolutions,
|
32
62
|
reference: str = "",
|
33
|
-
|
34
|
-
**kwargs: Unpack[ReviewKwargs[ReviewResult[str]]],
|
63
|
+
**kwargs: Unpack[ValidateKwargs[M]],
|
35
64
|
) -> Optional[M]:
|
36
|
-
"""
|
37
|
-
|
38
|
-
This method first conducts a review of the given object, then uses the review results
|
39
|
-
to generate a corrected version of the object using appropriate templates.
|
65
|
+
"""Fix a troubled object based on problem solutions.
|
40
66
|
|
41
67
|
Args:
|
42
|
-
obj (M): The object to be
|
68
|
+
obj (M): The object to be fixed.
|
69
|
+
problem_solutions (ProblemSolutions): The problem solutions to apply.
|
43
70
|
reference (str): A reference or contextual information for the object.
|
44
|
-
|
45
|
-
**kwargs: Review configuration parameters including criteria and review options.
|
71
|
+
**kwargs (Unpack[ValidateKwargs[M]]): Additional keyword arguments for the validation process.
|
46
72
|
|
47
73
|
Returns:
|
48
|
-
Optional[M]:
|
49
|
-
|
50
|
-
Raises:
|
51
|
-
TypeError: If the provided object doesn't implement Display or WithBriefing interfaces.
|
74
|
+
Optional[M]: The fixed object, or None if fixing fails.
|
52
75
|
"""
|
53
|
-
if not isinstance(obj, (Display, WithBriefing)):
|
54
|
-
raise TypeError(f"Expected Display or WithBriefing, got {type(obj)}")
|
55
|
-
|
56
|
-
review_res = await self.review_obj(obj, **kwargs)
|
57
|
-
if supervisor_check:
|
58
|
-
await review_res.supervisor_check()
|
59
|
-
if "default" in kwargs:
|
60
|
-
cast("ReviewKwargs[None]", kwargs)["default"] = None
|
61
76
|
return await self.propose(
|
62
|
-
obj.__class__,
|
77
|
+
cast("Type[M]", obj.__class__),
|
63
78
|
TEMPLATE_MANAGER.render_template(
|
64
|
-
configs.templates.
|
79
|
+
configs.templates.fix_troubled_obj_template,
|
65
80
|
{
|
66
|
-
"
|
67
|
-
"
|
81
|
+
"problem": problem_solutions.problem,
|
82
|
+
"solution": ok(
|
83
|
+
problem_solutions.final_solution(),
|
84
|
+
f"No solution found for problem: {problem_solutions.problem}",
|
85
|
+
),
|
86
|
+
"reference": reference,
|
68
87
|
},
|
69
88
|
),
|
70
89
|
**kwargs,
|
71
90
|
)
|
72
91
|
|
73
|
-
async def
|
74
|
-
self,
|
92
|
+
async def fix_troubled_string(
|
93
|
+
self,
|
94
|
+
input_text: str,
|
95
|
+
problem_solutions: ProblemSolutions,
|
96
|
+
reference: str = "",
|
97
|
+
**kwargs: Unpack[ValidateKwargs[str]],
|
75
98
|
) -> Optional[str]:
|
76
|
-
"""
|
77
|
-
|
78
|
-
This method applies the review process to the input text and generates
|
79
|
-
a corrected version based on the review results.
|
99
|
+
"""Fix a troubled string based on problem solutions.
|
80
100
|
|
81
101
|
Args:
|
82
|
-
input_text (str): The
|
83
|
-
|
84
|
-
|
102
|
+
input_text (str): The string to be fixed.
|
103
|
+
problem_solutions (ProblemSolutions): The problem solutions to apply.
|
104
|
+
reference (str): A reference or contextual information for the string.
|
105
|
+
**kwargs (Unpack[ValidateKwargs[str]]): Additional keyword arguments for the validation process.
|
85
106
|
|
86
107
|
Returns:
|
87
|
-
Optional[str]: The
|
108
|
+
Optional[str]: The fixed string, or None if fixing fails.
|
88
109
|
"""
|
89
|
-
review_res = await self.review_string(input_text, **kwargs)
|
90
|
-
if supervisor_check:
|
91
|
-
await review_res.supervisor_check()
|
92
|
-
|
93
|
-
if "default" in kwargs:
|
94
|
-
cast("ReviewKwargs[None]", kwargs)["default"] = None
|
95
110
|
return await self.ageneric_string(
|
96
111
|
TEMPLATE_MANAGER.render_template(
|
97
|
-
configs.templates.
|
112
|
+
configs.templates.fix_troubled_string_template,
|
113
|
+
{
|
114
|
+
"problem": problem_solutions.problem,
|
115
|
+
"solution": ok(
|
116
|
+
problem_solutions.final_solution(),
|
117
|
+
f"No solution found for problem: {problem_solutions.problem}",
|
118
|
+
),
|
119
|
+
"reference": reference,
|
120
|
+
"string_to_fix": input_text,
|
121
|
+
},
|
98
122
|
),
|
99
123
|
**kwargs,
|
100
124
|
)
|
101
125
|
|
102
|
-
async def
|
103
|
-
self,
|
104
|
-
|
105
|
-
|
126
|
+
async def correct_obj[M: SketchedAble](
|
127
|
+
self,
|
128
|
+
obj: M,
|
129
|
+
improvement: Improvement,
|
130
|
+
reference: str = "",
|
131
|
+
**kwargs: Unpack[ValidateKwargs[M]],
|
132
|
+
) -> Optional[M]:
|
133
|
+
"""Review and correct an object based on defined criteria and templates.
|
106
134
|
|
107
|
-
This
|
108
|
-
|
135
|
+
This method first conducts a review of the given object, then uses the review results
|
136
|
+
to generate a corrected version of the object using appropriate templates.
|
109
137
|
|
110
138
|
Args:
|
111
|
-
|
112
|
-
|
139
|
+
obj (M): The object to be reviewed and corrected. Must implement ProposedAble.
|
140
|
+
improvement (Improvement): The improvement object containing the review results.
|
141
|
+
reference (str): A reference or contextual information for the object.
|
142
|
+
**kwargs (Unpack[ValidateKwargs[M]]): Review configuration parameters including criteria and review options.
|
113
143
|
|
114
144
|
Returns:
|
115
|
-
Optional[
|
145
|
+
Optional[M]: A corrected version of the input object, or None if correction fails.
|
146
|
+
|
147
|
+
Raises:
|
148
|
+
TypeError: If the provided object doesn't implement Display or WithBriefing interfaces.
|
116
149
|
"""
|
117
|
-
|
150
|
+
if not improvement.decided():
|
151
|
+
improvement = await self.decide_improvement(improvement, **override_kwargs(kwargs, default=None))
|
152
|
+
|
153
|
+
for ps in improvement.problem_solutions:
|
154
|
+
fixed_obj = await self.fix_troubled_obj(obj, ps, reference, **kwargs)
|
155
|
+
if fixed_obj is None:
|
156
|
+
logger.error(
|
157
|
+
f"Failed to fix troubling obj {obj.__class__.__name__} when deal with problem: {ps.problem}",
|
158
|
+
)
|
159
|
+
return None
|
160
|
+
obj = fixed_obj
|
161
|
+
return obj
|
162
|
+
|
163
|
+
async def correct_string(
|
164
|
+
self, input_text: str, improvement: Improvement, reference: str = "", **kwargs: Unpack[ValidateKwargs[str]]
|
165
|
+
) -> Optional[str]:
|
166
|
+
"""Review and correct a string based on defined criteria and templates.
|
118
167
|
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
168
|
+
This method first conducts a review of the given string, then uses the review results
|
169
|
+
to generate a corrected version of the string using appropriate templates.
|
170
|
+
|
171
|
+
Args:
|
172
|
+
input_text (str): The string to be reviewed and corrected.
|
173
|
+
improvement (Improvement): The improvement object containing the review results.
|
174
|
+
reference (str): A reference or contextual information for the string.
|
175
|
+
**kwargs (Unpack[ValidateKwargs[str]]): Review configuration parameters including criteria and review options.
|
176
|
+
|
177
|
+
Returns:
|
178
|
+
Optional[str]: A corrected version of the input string, or None if correction fails.
|
179
|
+
"""
|
180
|
+
if not improvement.decided():
|
181
|
+
improvement = await self.decide_improvement(improvement, **override_kwargs(kwargs, default=None))
|
182
|
+
|
183
|
+
for ps in improvement.problem_solutions:
|
184
|
+
fixed_string = await self.fix_troubled_string(input_text, ps, reference, **kwargs)
|
185
|
+
if fixed_string is None:
|
186
|
+
logger.error(
|
187
|
+
f"Failed to fix troubling string when deal with problem: {ps.problem}",
|
188
|
+
)
|
189
|
+
return None
|
190
|
+
input_text = fixed_string
|
191
|
+
return input_text
|
192
|
+
|
193
|
+
async def correct_obj_inplace[M: ProposedUpdateAble](
|
194
|
+
self, obj: M, **kwargs: Unpack[CorrectKwargs[M]]
|
195
|
+
) -> Optional[M]:
|
196
|
+
"""Correct an object in place based on defined criteria and templates.
|
123
197
|
|
124
198
|
Args:
|
125
|
-
obj (M): The object to be
|
126
|
-
**kwargs (Unpack[
|
199
|
+
obj (M): The object to be corrected.
|
200
|
+
**kwargs (Unpack[CorrectKwargs[M]]): Additional keyword arguments for the correction process.
|
127
201
|
|
128
202
|
Returns:
|
129
|
-
M: The
|
203
|
+
Optional[M]: The corrected object, or None if correction fails.
|
130
204
|
"""
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
...
|
137
|
-
if (modified_obj := await self.correct_obj(
|
138
|
-
last_modified_obj,
|
139
|
-
topic=topic,
|
140
|
-
**kwargs,
|
141
|
-
)) is None:
|
142
|
-
break
|
143
|
-
last_modified_obj = modified_obj
|
144
|
-
rprint(last_modified_obj.finalized_dump())
|
145
|
-
return modified_obj or last_modified_obj
|
205
|
+
corrected_obj = await self.correct_obj(obj, **kwargs)
|
206
|
+
if corrected_obj is None:
|
207
|
+
return corrected_obj
|
208
|
+
obj.update_from(corrected_obj)
|
209
|
+
return obj
|
@@ -10,28 +10,43 @@ from fabricatio.models.usages import LLMUsage
|
|
10
10
|
class Propose(LLMUsage):
|
11
11
|
"""A class that proposes an Obj based on a prompt."""
|
12
12
|
|
13
|
+
@overload
|
14
|
+
async def propose[M: ProposedAble](
|
15
|
+
self,
|
16
|
+
cls: Type[M],
|
17
|
+
prompt: List[str],
|
18
|
+
**kwargs: Unpack[ValidateKwargs[None]],
|
19
|
+
) -> List[Optional[M]]: ...
|
20
|
+
|
13
21
|
@overload
|
14
22
|
async def propose[M: ProposedAble](
|
15
23
|
self,
|
16
24
|
cls: Type[M],
|
17
25
|
prompt: List[str],
|
18
26
|
**kwargs: Unpack[ValidateKwargs[M]],
|
19
|
-
) ->
|
27
|
+
) -> List[M]: ...
|
20
28
|
|
21
29
|
@overload
|
22
30
|
async def propose[M: ProposedAble](
|
23
31
|
self,
|
24
32
|
cls: Type[M],
|
25
33
|
prompt: str,
|
26
|
-
**kwargs: Unpack[ValidateKwargs[
|
34
|
+
**kwargs: Unpack[ValidateKwargs[None]],
|
27
35
|
) -> Optional[M]: ...
|
36
|
+
@overload
|
37
|
+
async def propose[M: ProposedAble](
|
38
|
+
self,
|
39
|
+
cls: Type[M],
|
40
|
+
prompt: str,
|
41
|
+
**kwargs: Unpack[ValidateKwargs[M]],
|
42
|
+
) -> M: ...
|
28
43
|
|
29
44
|
async def propose[M: ProposedAble](
|
30
45
|
self,
|
31
46
|
cls: Type[M],
|
32
47
|
prompt: List[str] | str,
|
33
|
-
**kwargs: Unpack[ValidateKwargs[M]],
|
34
|
-
) -> Optional[List[M] | M]:
|
48
|
+
**kwargs: Unpack[ValidateKwargs[Optional[M]]],
|
49
|
+
) -> Optional[M] | List[Optional[M]] | M | List[M]:
|
35
50
|
"""Asynchronously proposes a task based on a given prompt and parameters.
|
36
51
|
|
37
52
|
Parameters:
|
@@ -47,3 +62,4 @@ class Propose(LLMUsage):
|
|
47
62
|
validator=cls.instantiate_from_string,
|
48
63
|
**kwargs,
|
49
64
|
)
|
65
|
+
|
fabricatio/capabilities/rag.py
CHANGED
@@ -3,14 +3,13 @@
|
|
3
3
|
try:
|
4
4
|
from pymilvus import MilvusClient
|
5
5
|
except ImportError as e:
|
6
|
-
raise RuntimeError("pymilvus is not installed. Have you installed `fabricatio[rag]` instead of `fabricatio
|
6
|
+
raise RuntimeError("pymilvus is not installed. Have you installed `fabricatio[rag]` instead of `fabricatio`?") from e
|
7
7
|
from functools import lru_cache
|
8
8
|
from operator import itemgetter
|
9
9
|
from os import PathLike
|
10
10
|
from pathlib import Path
|
11
11
|
from typing import Any, Callable, Dict, List, Optional, Self, Union, Unpack, cast, overload
|
12
12
|
|
13
|
-
from fabricatio._rust_instances import TEMPLATE_MANAGER
|
14
13
|
from fabricatio.config import configs
|
15
14
|
from fabricatio.journal import logger
|
16
15
|
from fabricatio.models.kwargs_types import (
|
@@ -22,7 +21,9 @@ from fabricatio.models.kwargs_types import (
|
|
22
21
|
RetrievalKwargs,
|
23
22
|
)
|
24
23
|
from fabricatio.models.usages import EmbeddingUsage
|
25
|
-
from fabricatio.models.utils import MilvusData
|
24
|
+
from fabricatio.models.utils import MilvusData
|
25
|
+
from fabricatio.rust_instances import TEMPLATE_MANAGER
|
26
|
+
from fabricatio.utils import ok
|
26
27
|
from more_itertools.recipes import flatten, unique
|
27
28
|
from pydantic import Field, PrivateAttr
|
28
29
|
|
@@ -376,7 +377,7 @@ class RAG(EmbeddingUsage):
|
|
376
377
|
Returns:
|
377
378
|
List[str]: A list of refined questions.
|
378
379
|
"""
|
379
|
-
return await self.
|
380
|
+
return await self.alist_str(
|
380
381
|
TEMPLATE_MANAGER.render_template(
|
381
382
|
configs.templates.refined_query_template,
|
382
383
|
{"question": [question] if isinstance(question, str) else question},
|
@@ -4,19 +4,19 @@ from itertools import permutations
|
|
4
4
|
from random import sample
|
5
5
|
from typing import Dict, List, Optional, Set, Tuple, Union, Unpack, overload
|
6
6
|
|
7
|
-
from fabricatio._rust_instances import TEMPLATE_MANAGER
|
8
7
|
from fabricatio.config import configs
|
9
8
|
from fabricatio.journal import logger
|
10
|
-
from fabricatio.models.generic import
|
11
|
-
from fabricatio.models.kwargs_types import ValidateKwargs
|
9
|
+
from fabricatio.models.generic import Display
|
10
|
+
from fabricatio.models.kwargs_types import CompositeScoreKwargs, ValidateKwargs
|
12
11
|
from fabricatio.models.usages import LLMUsage
|
13
|
-
from fabricatio.models.utils import override_kwargs
|
14
12
|
from fabricatio.parser import JsonCapture
|
13
|
+
from fabricatio.rust_instances import TEMPLATE_MANAGER
|
14
|
+
from fabricatio.utils import ok, override_kwargs
|
15
15
|
from more_itertools import flatten, windowed
|
16
16
|
from pydantic import NonNegativeInt, PositiveInt
|
17
17
|
|
18
18
|
|
19
|
-
class
|
19
|
+
class Rating(LLMUsage):
|
20
20
|
"""A class that provides functionality to rate tasks based on a rating manual and score range.
|
21
21
|
|
22
22
|
References:
|
@@ -87,6 +87,7 @@ class GiveRating(WithBriefing, LLMUsage):
|
|
87
87
|
to_rate: str,
|
88
88
|
topic: str,
|
89
89
|
criteria: Set[str],
|
90
|
+
manual: Optional[Dict[str, str]],
|
90
91
|
score_range: Tuple[float, float] = (0.0, 1.0),
|
91
92
|
**kwargs: Unpack[ValidateKwargs],
|
92
93
|
) -> Dict[str, float]: ...
|
@@ -97,6 +98,7 @@ class GiveRating(WithBriefing, LLMUsage):
|
|
97
98
|
to_rate: List[str],
|
98
99
|
topic: str,
|
99
100
|
criteria: Set[str],
|
101
|
+
manual: Optional[Dict[str, str]],
|
100
102
|
score_range: Tuple[float, float] = (0.0, 1.0),
|
101
103
|
**kwargs: Unpack[ValidateKwargs],
|
102
104
|
) -> List[Dict[str, float]]: ...
|
@@ -106,6 +108,7 @@ class GiveRating(WithBriefing, LLMUsage):
|
|
106
108
|
to_rate: Union[str, List[str]],
|
107
109
|
topic: str,
|
108
110
|
criteria: Set[str],
|
111
|
+
manual: Optional[Dict[str, str]],
|
109
112
|
score_range: Tuple[float, float] = (0.0, 1.0),
|
110
113
|
**kwargs: Unpack[ValidateKwargs],
|
111
114
|
) -> Optional[Dict[str, float] | List[Dict[str, float]]]:
|
@@ -115,6 +118,7 @@ class GiveRating(WithBriefing, LLMUsage):
|
|
115
118
|
to_rate (Union[str, List[str]]): The string or sequence of strings to be rated.
|
116
119
|
topic (str): The topic related to the task.
|
117
120
|
criteria (Set[str]): A set of criteria for rating.
|
121
|
+
manual (Optional[Dict[str, str]]): A dictionary containing the rating criteria. If not provided, then this method will draft the criteria automatically.
|
118
122
|
score_range (Tuple[float, float], optional): A tuple representing the valid score range. Defaults to (0.0, 1.0).
|
119
123
|
**kwargs (Unpack[ValidateKwargs]): Additional keyword arguments for the LLM usage.
|
120
124
|
|
@@ -122,7 +126,11 @@ class GiveRating(WithBriefing, LLMUsage):
|
|
122
126
|
Union[Dict[str, float], List[Dict[str, float]]]: A dictionary with the ratings for each criterion if a single string is provided,
|
123
127
|
or a list of dictionaries with the ratings for each criterion if a sequence of strings is provided.
|
124
128
|
"""
|
125
|
-
manual =
|
129
|
+
manual = (
|
130
|
+
manual
|
131
|
+
or await self.draft_rating_manual(topic, criteria, **override_kwargs(kwargs, default=None))
|
132
|
+
or dict(zip(criteria, criteria, strict=True))
|
133
|
+
)
|
126
134
|
|
127
135
|
return await self.rate_fine_grind(to_rate, manual, score_range, **kwargs)
|
128
136
|
|
@@ -149,9 +157,7 @@ class GiveRating(WithBriefing, LLMUsage):
|
|
149
157
|
return json_data
|
150
158
|
return None
|
151
159
|
|
152
|
-
criteria = criteria or await self.draft_rating_criteria(
|
153
|
-
topic, **self.prepend_sys_msg(override_kwargs(dict(kwargs), default=None))
|
154
|
-
)
|
160
|
+
criteria = criteria or await self.draft_rating_criteria(topic, **override_kwargs(dict(kwargs), default=None))
|
155
161
|
|
156
162
|
if criteria is None:
|
157
163
|
logger.error(f"Failed to draft rating criteria for topic {topic}")
|
@@ -168,7 +174,7 @@ class GiveRating(WithBriefing, LLMUsage):
|
|
168
174
|
)
|
169
175
|
),
|
170
176
|
validator=_validator,
|
171
|
-
**
|
177
|
+
**kwargs,
|
172
178
|
)
|
173
179
|
|
174
180
|
async def draft_rating_criteria(
|
@@ -200,7 +206,7 @@ class GiveRating(WithBriefing, LLMUsage):
|
|
200
206
|
validator=lambda resp: set(out)
|
201
207
|
if (out := JsonCapture.validate_with(resp, list, str, criteria_count)) is not None
|
202
208
|
else out,
|
203
|
-
**
|
209
|
+
**kwargs,
|
204
210
|
)
|
205
211
|
|
206
212
|
async def draft_rating_criteria_from_examples(
|
@@ -253,7 +259,7 @@ class GiveRating(WithBriefing, LLMUsage):
|
|
253
259
|
validator=lambda resp: JsonCapture.validate_with(
|
254
260
|
resp, target_type=list, elements_type=str, length=reasons_count
|
255
261
|
),
|
256
|
-
**
|
262
|
+
**kwargs,
|
257
263
|
)
|
258
264
|
)
|
259
265
|
# extract certain mount of criteria from reasons according to their importance and frequency
|
@@ -310,9 +316,9 @@ class GiveRating(WithBriefing, LLMUsage):
|
|
310
316
|
for pair in windows
|
311
317
|
],
|
312
318
|
validator=lambda resp: JsonCapture.validate_with(resp, target_type=float),
|
313
|
-
**
|
319
|
+
**kwargs,
|
314
320
|
)
|
315
|
-
weights = [1]
|
321
|
+
weights = [1.0]
|
316
322
|
for rw in relative_weights:
|
317
323
|
weights.append(weights[-1] * rw)
|
318
324
|
total = sum(weights)
|
@@ -322,27 +328,66 @@ class GiveRating(WithBriefing, LLMUsage):
|
|
322
328
|
self,
|
323
329
|
topic: str,
|
324
330
|
to_rate: List[str],
|
325
|
-
|
326
|
-
|
327
|
-
|
331
|
+
criteria: Optional[Set[str]] = None,
|
332
|
+
weights: Optional[Dict[str, float]] = None,
|
333
|
+
manual: Optional[Dict[str, str]] = None,
|
334
|
+
**kwargs: Unpack[ValidateKwargs[List[Dict[str, float]]]],
|
328
335
|
) -> List[float]:
|
329
336
|
"""Calculates the composite scores for a list of items based on a given topic and criteria.
|
330
337
|
|
331
338
|
Args:
|
332
339
|
topic (str): The topic for the rating.
|
333
340
|
to_rate (List[str]): A list of strings to be rated.
|
334
|
-
|
335
|
-
|
341
|
+
criteria (Optional[Set[str]]): A set of criteria for the rating. Defaults to None.
|
342
|
+
weights (Optional[Dict[str, float]]): A dictionary of rating weights for each criterion. Defaults to None.
|
343
|
+
manual (Optional[Dict[str, str]]): A dictionary of manual ratings for each item. Defaults to None.
|
336
344
|
**kwargs (Unpack[ValidateKwargs]): Additional keyword arguments for the LLM usage.
|
337
345
|
|
338
346
|
Returns:
|
339
347
|
List[float]: A list of composite scores for the items.
|
340
348
|
"""
|
341
|
-
criteria =
|
342
|
-
|
349
|
+
criteria = ok(
|
350
|
+
criteria
|
351
|
+
or await self.draft_rating_criteria_from_examples(topic, to_rate, **override_kwargs(kwargs, default=None))
|
352
|
+
)
|
353
|
+
weights = ok(
|
354
|
+
weights or await self.drafting_rating_weights_klee(topic, criteria, **override_kwargs(kwargs, default=None))
|
343
355
|
)
|
344
|
-
weights = await self.drafting_rating_weights_klee(topic, criteria, **kwargs)
|
345
356
|
logger.info(f"Criteria: {criteria}\nWeights: {weights}")
|
346
|
-
ratings_seq = await self.rate(to_rate, topic, criteria, **kwargs)
|
357
|
+
ratings_seq = await self.rate(to_rate, topic, criteria, manual, **kwargs)
|
347
358
|
|
348
359
|
return [sum(ratings[c] * weights[c] for c in criteria) for ratings in ratings_seq]
|
360
|
+
|
361
|
+
@overload
|
362
|
+
async def best(self, candidates: List[str], k: int=1, **kwargs: Unpack[CompositeScoreKwargs]) -> List[str]: ...
|
363
|
+
@overload
|
364
|
+
async def best[T: Display](
|
365
|
+
self, candidates: List[T], k: int=1, **kwargs: Unpack[CompositeScoreKwargs]
|
366
|
+
) -> List[T]: ...
|
367
|
+
|
368
|
+
async def best[T: Display](
|
369
|
+
self, candidates: List[str] | List[T], k: int=1, **kwargs: Unpack[CompositeScoreKwargs]
|
370
|
+
) -> Optional[List[str] | List[T]]:
|
371
|
+
"""Choose the best candidates from the list of candidates based on the composite score.
|
372
|
+
|
373
|
+
Args:
|
374
|
+
k (int): The number of best candidates to choose.
|
375
|
+
candidates (List[str]): A list of candidates to choose from.
|
376
|
+
**kwargs (CompositeScoreKwargs): Additional keyword arguments for the composite score calculation.
|
377
|
+
|
378
|
+
Returns:
|
379
|
+
List[str]: The best candidates.
|
380
|
+
"""
|
381
|
+
if (leng := len(candidates)) == 0:
|
382
|
+
logger.warning(f"No candidates, got {leng}, return None.")
|
383
|
+
return None
|
384
|
+
|
385
|
+
if leng == 1:
|
386
|
+
logger.warning(f"Only one candidate, got {leng}, return it.")
|
387
|
+
return candidates
|
388
|
+
logger.info(f"Choose best {k} from {leng} candidates.")
|
389
|
+
|
390
|
+
rating_seq = await self.composite_score(
|
391
|
+
to_rate=[c.display() if isinstance(c, Display) else c for c in candidates], **kwargs
|
392
|
+
)
|
393
|
+
return [a[0] for a in sorted(zip(candidates, rating_seq, strict=True), key=lambda x: x[1], reverse=True)[:k]] # pyright: ignore [reportReturnType]
|