fabricatio 0.2.1.dev2__tar.gz → 0.2.1.dev4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/PKG-INFO +2 -2
- fabricatio-0.2.1.dev4/examples/make_a_rating/rating.py +103 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/examples/minor/hello_fabricatio.py +1 -1
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/examples/propose_task/propose.py +1 -1
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/examples/simple_chat/chat.py +1 -1
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/pyproject.toml +2 -2
- fabricatio-0.2.1.dev4/python/fabricatio/capabilities/rating.py +277 -0
- fabricatio-0.2.1.dev2/python/fabricatio/models/advanced.py → fabricatio-0.2.1.dev4/python/fabricatio/capabilities/task.py +2 -115
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/python/fabricatio/config.py +9 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/python/fabricatio/models/action.py +3 -2
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/python/fabricatio/models/kwargs_types.py +7 -2
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/python/fabricatio/models/role.py +3 -3
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/python/fabricatio/models/task.py +7 -4
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/python/fabricatio/models/usages.py +41 -45
- fabricatio-0.2.1.dev4/templates/built-in/draft_rating_criteria.hbs +24 -0
- fabricatio-0.2.1.dev4/templates/built-in/draft_rating_manual.hbs +27 -0
- fabricatio-0.2.1.dev4/templates/built-in/extract_criteria_from_reasons.hbs +24 -0
- fabricatio-0.2.1.dev4/templates/built-in/extract_reasons_from_examples.hbs +18 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/templates/built-in/propose_task.hbs +3 -2
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/templates/built-in/rate_fine_grind.hbs +4 -4
- fabricatio-0.2.1.dev4/templates.tar.gz +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/tests/test_models/test_advanced.py +6 -3
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/uv.lock +12 -21
- fabricatio-0.2.1.dev2/templates/built-in/draft_rating_manual.hbs +0 -26
- fabricatio-0.2.1.dev2/templates.tar.gz +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/.github/workflows/build-package.yaml +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/.github/workflows/ruff.yaml +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/.github/workflows/tests.yaml +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/.gitignore +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/.python-version +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/Cargo.lock +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/Cargo.toml +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/LICENSE +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/Makefile +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/README.md +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/examples/llm_usages/llm_usage.py +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/examples/make_diary/commits.json +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/examples/make_diary/diary.py +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/examples/task_handle/handle_task.py +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/python/fabricatio/__init__.py +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/python/fabricatio/_rust.pyi +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/python/fabricatio/_rust_instances.py +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/python/fabricatio/actions/__init__.py +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/python/fabricatio/actions/communication.py +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/python/fabricatio/actions/transmission.py +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/python/fabricatio/core.py +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/python/fabricatio/decorators.py +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/python/fabricatio/fs/__init__.py +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/python/fabricatio/fs/curd.py +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/python/fabricatio/fs/readers.py +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/python/fabricatio/journal.py +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/python/fabricatio/models/events.py +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/python/fabricatio/models/generic.py +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/python/fabricatio/models/tool.py +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/python/fabricatio/models/utils.py +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/python/fabricatio/parser.py +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/python/fabricatio/py.typed +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/python/fabricatio/toolboxes/__init__.py +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/python/fabricatio/toolboxes/arithmetic.py +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/python/fabricatio/toolboxes/fs.py +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/python/fabricatio/toolboxes/task.py +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/src/hash.rs +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/src/lib.rs +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/src/templates.rs +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/templates/built-in/binary-exploitation-ctf-solver.hbs +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/templates/built-in/claude-xml.hbs +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/templates/built-in/clean-up-code.hbs +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/templates/built-in/cryptography-ctf-solver.hbs +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/templates/built-in/dependencies.hbs +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/templates/built-in/document-the-code.hbs +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/templates/built-in/draft_tool_usage_code.hbs +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/templates/built-in/find-security-vulnerabilities.hbs +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/templates/built-in/fix-bugs.hbs +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/templates/built-in/improve-performance.hbs +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/templates/built-in/make_choice.hbs +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/templates/built-in/make_judgment.hbs +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/templates/built-in/refactor.hbs +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/templates/built-in/reverse-engineering-ctf-solver.hbs +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/templates/built-in/task_briefing.hbs +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/templates/built-in/web-ctf-solver.hbs +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/templates/built-in/write-git-commit.hbs +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/templates/built-in/write-github-pull-request.hbs +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/templates/built-in/write-github-readme.hbs +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/tests/test_config.py +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/tests/test_models/test_action.py +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/tests/test_models/test_generic.py +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/tests/test_models/test_role.py +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/tests/test_models/test_task.py +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/tests/test_models/test_tool.py +0 -0
- {fabricatio-0.2.1.dev2 → fabricatio-0.2.1.dev4}/tests/test_models/test_usages.py +0 -0
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: fabricatio
|
3
|
-
Version: 0.2.1.
|
3
|
+
Version: 0.2.1.dev4
|
4
4
|
Classifier: License :: OSI Approved :: MIT License
|
5
5
|
Classifier: Programming Language :: Rust
|
6
6
|
Classifier: Programming Language :: Python :: 3.12
|
@@ -15,6 +15,7 @@ Requires-Dist: gitpython>=3.1.44
|
|
15
15
|
Requires-Dist: litellm>=1.60.0
|
16
16
|
Requires-Dist: loguru>=0.7.3
|
17
17
|
Requires-Dist: magika>=0.5.1
|
18
|
+
Requires-Dist: more-itertools>=10.6.0
|
18
19
|
Requires-Dist: orjson>=3.10.15
|
19
20
|
Requires-Dist: pydantic>=2.10.6
|
20
21
|
Requires-Dist: pydantic-settings>=2.7.1
|
@@ -22,7 +23,6 @@ Requires-Dist: pymitter>=1.0.0
|
|
22
23
|
Requires-Dist: questionary>=2.1.0
|
23
24
|
Requires-Dist: regex>=2024.11.6
|
24
25
|
Requires-Dist: rich>=13.9.4
|
25
|
-
Requires-Dist: faiss-cpu>=1.10.0 ; extra == 'rag'
|
26
26
|
Requires-Dist: pymilvus>=2.5.4 ; extra == 'rag'
|
27
27
|
Requires-Dist: fabricatio[rag] ; extra == 'full'
|
28
28
|
Provides-Extra: rag
|
@@ -0,0 +1,103 @@
|
|
1
|
+
"""Example of proposing a task to a role."""
|
2
|
+
|
3
|
+
import asyncio
|
4
|
+
from typing import Dict, List, Set, Unpack
|
5
|
+
|
6
|
+
import orjson
|
7
|
+
from fabricatio import Action, JsonCapture, Role, WorkFlow, logger
|
8
|
+
from fabricatio.models.events import Event
|
9
|
+
from fabricatio.models.task import Task
|
10
|
+
|
11
|
+
|
12
|
+
class Rate(Action):
|
13
|
+
"""Rate the task."""
|
14
|
+
|
15
|
+
name: str = "rate"
|
16
|
+
output_key: str = "task_output"
|
17
|
+
|
18
|
+
async def _execute(self, to_rate: List[str], rate_topic: str, criteria: Set[str], **_) -> List[Dict[str, float]]:
|
19
|
+
logger.info(f"Rating the: \n{to_rate}")
|
20
|
+
"""Rate the task."""
|
21
|
+
return await self.rate(
|
22
|
+
to_rate,
|
23
|
+
rate_topic,
|
24
|
+
criteria,
|
25
|
+
)
|
26
|
+
|
27
|
+
|
28
|
+
class WhatToRate(Action):
|
29
|
+
"""Figure out what to rate."""
|
30
|
+
|
31
|
+
name: str = "figure out what to rate"
|
32
|
+
|
33
|
+
output_key: str = "to_rate"
|
34
|
+
|
35
|
+
async def _execute(self, task_input: Task, rate_topic: str, **cxt: Unpack) -> List[str]:
|
36
|
+
def _validate(resp: str) -> List[str] | None:
|
37
|
+
if (
|
38
|
+
(cap := JsonCapture.convert_with(resp, orjson.loads)) is not None
|
39
|
+
and isinstance(cap, list)
|
40
|
+
and all(isinstance(i, str) for i in cap)
|
41
|
+
):
|
42
|
+
return cap
|
43
|
+
return None
|
44
|
+
|
45
|
+
return await self.aask_validate(
|
46
|
+
f"This is task briefing:\n{task_input.briefing}\n\n"
|
47
|
+
f"We are talking about {rate_topic}. you need to extract targets to rate into a the JSON array\n"
|
48
|
+
f"The response SHALL be a JSON array of strings within the codeblock\n"
|
49
|
+
f"# Example\n"
|
50
|
+
f'```json\n["this is a target to rate", "this is another target to rate"]\n```',
|
51
|
+
_validate,
|
52
|
+
)
|
53
|
+
|
54
|
+
|
55
|
+
class MakeCriteria(Action):
|
56
|
+
"""Make criteria for rating."""
|
57
|
+
|
58
|
+
name: str = "make criteria"
|
59
|
+
output_key: str = "criteria"
|
60
|
+
|
61
|
+
async def _execute(self, task_input: Task, rate_topic: str, to_rate: List[str], **cxt: Unpack) -> Set[str]:
|
62
|
+
criteria = await self.draft_rating_criteria_from_examples(rate_topic, to_rate)
|
63
|
+
logger.info(f"Criteria: \n{criteria}")
|
64
|
+
return set(criteria)
|
65
|
+
|
66
|
+
|
67
|
+
async def main() -> None:
|
68
|
+
"""Main function."""
|
69
|
+
role = Role(
|
70
|
+
name="TaskRater",
|
71
|
+
description="A role that can rate tasks.",
|
72
|
+
registry={
|
73
|
+
Event.instantiate_from("rate_food").push_wildcard().push("pending"): WorkFlow(
|
74
|
+
name="Rate food",
|
75
|
+
steps=(WhatToRate, Rate),
|
76
|
+
extra_init_context={
|
77
|
+
"rate_topic": "If this food is cheap and delicious",
|
78
|
+
"criteria": {"taste", "price", "quality", "safety", "healthiness"},
|
79
|
+
},
|
80
|
+
),
|
81
|
+
Event.instantiate_from("make_criteria_for_food").push_wildcard().push("pending"): WorkFlow(
|
82
|
+
name="Make criteria for food",
|
83
|
+
steps=(WhatToRate, MakeCriteria, Rate),
|
84
|
+
extra_init_context={
|
85
|
+
"rate_topic": "if the food is 'good'",
|
86
|
+
},
|
87
|
+
),
|
88
|
+
},
|
89
|
+
)
|
90
|
+
task = await role.propose(
|
91
|
+
"rate these food, so that i can decide what to eat today. choco cake, strawberry icecream, giga burger, cup of coffee, rotten bread from the trash bin, and a salty of fruit salad",
|
92
|
+
)
|
93
|
+
rating = await task.move_to("rate_food").delegate()
|
94
|
+
|
95
|
+
logger.success(f"Result: \n{rating}")
|
96
|
+
|
97
|
+
generated_criteria = await task.move_to("make_criteria_for_food").delegate()
|
98
|
+
|
99
|
+
logger.success(f"Generated Criteria: \n{generated_criteria}")
|
100
|
+
|
101
|
+
|
102
|
+
if __name__ == "__main__":
|
103
|
+
asyncio.run(main())
|
@@ -5,7 +5,7 @@ from typing import Any
|
|
5
5
|
|
6
6
|
from fabricatio import Action, Role, Task, WorkFlow, logger
|
7
7
|
|
8
|
-
task = Task(name="say hello", goal="say hello", description="say hello to the world")
|
8
|
+
task = Task(name="say hello", goal=["say hello"], description="say hello to the world")
|
9
9
|
|
10
10
|
|
11
11
|
class Hello(Action):
|
@@ -5,7 +5,7 @@ from typing import Any
|
|
5
5
|
|
6
6
|
from fabricatio import Action, Role, Task, WorkFlow, logger
|
7
7
|
|
8
|
-
task = Task(name="say hello", goal="say hello", description="say hello to the world")
|
8
|
+
task = Task(name="say hello", goal=["say hello"], description="say hello to the world")
|
9
9
|
|
10
10
|
|
11
11
|
class Talk(Action):
|
@@ -5,7 +5,7 @@ from typing import Any
|
|
5
5
|
|
6
6
|
from fabricatio import Action, Role, Task, WorkFlow, logger
|
7
7
|
|
8
|
-
task = Task(name="say hello", goal="say hello", description="say hello to the world")
|
8
|
+
task = Task(name="say hello", goal=["say hello"], description="say hello to the world")
|
9
9
|
|
10
10
|
|
11
11
|
class Talk(Action):
|
@@ -1,6 +1,6 @@
|
|
1
1
|
[project]
|
2
2
|
name = "fabricatio"
|
3
|
-
version = "0.2.1-
|
3
|
+
version = "0.2.1-dev4"
|
4
4
|
description = "A LLM multi-agent framework."
|
5
5
|
readme = "README.md"
|
6
6
|
license = { file = "LICENSE" }
|
@@ -34,6 +34,7 @@ dependencies = [
|
|
34
34
|
"litellm>=1.60.0",
|
35
35
|
"loguru>=0.7.3",
|
36
36
|
"magika>=0.5.1",
|
37
|
+
"more-itertools>=10.6.0",
|
37
38
|
"orjson>=3.10.15",
|
38
39
|
"pydantic>=2.10.6",
|
39
40
|
"pydantic-settings>=2.7.1",
|
@@ -76,7 +77,6 @@ dev = [
|
|
76
77
|
|
77
78
|
[project.optional-dependencies]
|
78
79
|
rag = [
|
79
|
-
"faiss-cpu>=1.10.0",
|
80
80
|
"pymilvus>=2.5.4",
|
81
81
|
]
|
82
82
|
|
@@ -0,0 +1,277 @@
|
|
1
|
+
"""A module that provides functionality to rate tasks based on a rating manual and score range."""
|
2
|
+
|
3
|
+
from asyncio import gather
|
4
|
+
from itertools import permutations
|
5
|
+
from typing import Dict, List, Set, Tuple, Union, Unpack, overload
|
6
|
+
|
7
|
+
import orjson
|
8
|
+
from fabricatio._rust_instances import template_manager
|
9
|
+
from fabricatio.config import configs
|
10
|
+
from fabricatio.journal import logger
|
11
|
+
from fabricatio.models.generic import WithBriefing
|
12
|
+
from fabricatio.models.kwargs_types import GenerateKwargs, ValidateKwargs
|
13
|
+
from fabricatio.models.usages import LLMUsage
|
14
|
+
from fabricatio.parser import JsonCapture
|
15
|
+
from more_itertools import flatten
|
16
|
+
from pydantic import NonNegativeInt, PositiveInt
|
17
|
+
|
18
|
+
|
19
|
+
class GiveRating(WithBriefing, LLMUsage):
|
20
|
+
"""A class that provides functionality to rate tasks based on a rating manual and score range."""
|
21
|
+
|
22
|
+
async def rate_fine_grind(
|
23
|
+
self,
|
24
|
+
to_rate: str,
|
25
|
+
rating_manual: Dict[str, str],
|
26
|
+
score_range: Tuple[float, float],
|
27
|
+
**kwargs: Unpack[ValidateKwargs],
|
28
|
+
) -> Dict[str, float]:
|
29
|
+
"""Rate a given string based on a rating manual and score range.
|
30
|
+
|
31
|
+
Args:
|
32
|
+
to_rate (str): The string to be rated.
|
33
|
+
rating_manual (Dict[str, str]): A dictionary containing the rating criteria.
|
34
|
+
score_range (Tuple[float, float]): A tuple representing the valid score range.
|
35
|
+
**kwargs (Unpack[ValidateKwargs]): Additional keyword arguments for the LLM usage.
|
36
|
+
|
37
|
+
Returns:
|
38
|
+
Dict[str, float]: A dictionary with the ratings for each dimension.
|
39
|
+
"""
|
40
|
+
|
41
|
+
def _validator(response: str) -> Dict[str, float] | None:
|
42
|
+
if (
|
43
|
+
(json_data := JsonCapture.convert_with(response, orjson.loads)) is not None
|
44
|
+
and isinstance(json_data, dict)
|
45
|
+
and json_data.keys() == rating_manual.keys()
|
46
|
+
and all(isinstance(v, float) for v in json_data.values())
|
47
|
+
and all(score_range[0] <= v <= score_range[1] for v in json_data.values())
|
48
|
+
):
|
49
|
+
return json_data
|
50
|
+
return None
|
51
|
+
|
52
|
+
logger.info(f"Rating for {to_rate}")
|
53
|
+
return await self.aask_validate(
|
54
|
+
question=(
|
55
|
+
template_manager.render_template(
|
56
|
+
configs.templates.rate_fine_grind_template,
|
57
|
+
{
|
58
|
+
"to_rate": to_rate,
|
59
|
+
"min_score": score_range[0],
|
60
|
+
"max_score": score_range[1],
|
61
|
+
"rating_manual": rating_manual,
|
62
|
+
},
|
63
|
+
)
|
64
|
+
),
|
65
|
+
validator=_validator,
|
66
|
+
system_message=f"# your personal briefing: \n{self.briefing}",
|
67
|
+
**kwargs,
|
68
|
+
)
|
69
|
+
|
70
|
+
@overload
|
71
|
+
async def rate(
|
72
|
+
self,
|
73
|
+
to_rate: str,
|
74
|
+
topic: str,
|
75
|
+
criteria: Set[str],
|
76
|
+
score_range: Tuple[float, float] = (0.0, 1.0),
|
77
|
+
**kwargs: Unpack[ValidateKwargs],
|
78
|
+
) -> Dict[str, float]: ...
|
79
|
+
|
80
|
+
@overload
|
81
|
+
async def rate(
|
82
|
+
self,
|
83
|
+
to_rate: List[str],
|
84
|
+
topic: str,
|
85
|
+
criteria: Set[str],
|
86
|
+
score_range: Tuple[float, float] = (0.0, 1.0),
|
87
|
+
**kwargs: Unpack[ValidateKwargs],
|
88
|
+
) -> List[Dict[str, float]]: ...
|
89
|
+
|
90
|
+
async def rate(
|
91
|
+
self,
|
92
|
+
to_rate: Union[str, List[str]],
|
93
|
+
topic: str,
|
94
|
+
criteria: Set[str],
|
95
|
+
score_range: Tuple[float, float] = (0.0, 1.0),
|
96
|
+
**kwargs: Unpack[ValidateKwargs],
|
97
|
+
) -> Union[Dict[str, float], List[Dict[str, float]]]:
|
98
|
+
"""Rate a given string or a sequence of strings based on a topic, criteria, and score range.
|
99
|
+
|
100
|
+
Args:
|
101
|
+
to_rate (Union[str, List[str]]): The string or sequence of strings to be rated.
|
102
|
+
topic (str): The topic related to the task.
|
103
|
+
criteria (Set[str]): A set of criteria for rating.
|
104
|
+
score_range (Tuple[float, float], optional): A tuple representing the valid score range. Defaults to (0.0, 1.0).
|
105
|
+
**kwargs (Unpack[ValidateKwargs]): Additional keyword arguments for the LLM usage.
|
106
|
+
|
107
|
+
Returns:
|
108
|
+
Union[Dict[str, float], List[Dict[str, float]]]: A dictionary with the ratings for each criterion if a single string is provided,
|
109
|
+
or a list of dictionaries with the ratings for each criterion if a sequence of strings is provided.
|
110
|
+
"""
|
111
|
+
manual = await self.draft_rating_manual(topic, criteria, **kwargs)
|
112
|
+
if isinstance(to_rate, str):
|
113
|
+
return await self.rate_fine_grind(to_rate, manual, score_range, **kwargs)
|
114
|
+
if isinstance(to_rate, list):
|
115
|
+
return await gather(*[self.rate_fine_grind(item, manual, score_range, **kwargs) for item in to_rate])
|
116
|
+
raise ValueError("to_rate must be a string or a list of strings")
|
117
|
+
|
118
|
+
async def draft_rating_manual(
|
119
|
+
self, topic: str, criteria: Set[str], **kwargs: Unpack[ValidateKwargs]
|
120
|
+
) -> Dict[str, str]:
|
121
|
+
"""Drafts a rating manual based on a topic and dimensions.
|
122
|
+
|
123
|
+
Args:
|
124
|
+
topic (str): The topic for the rating manual.
|
125
|
+
criteria (Set[str]): A set of dimensions for the rating manual.
|
126
|
+
**kwargs (Unpack[ValidateKwargs]): Additional keyword arguments for the LLM usage.
|
127
|
+
|
128
|
+
Returns:
|
129
|
+
Dict[str, str]: A dictionary representing the drafted rating manual.
|
130
|
+
"""
|
131
|
+
|
132
|
+
def _validator(response: str) -> Dict[str, str] | None:
|
133
|
+
if (
|
134
|
+
(json_data := JsonCapture.convert_with(response, orjson.loads)) is not None
|
135
|
+
and isinstance(json_data, dict)
|
136
|
+
and json_data.keys() == criteria
|
137
|
+
and all(isinstance(v, str) for v in json_data.values())
|
138
|
+
):
|
139
|
+
return json_data
|
140
|
+
return None
|
141
|
+
|
142
|
+
return await self.aask_validate(
|
143
|
+
question=(
|
144
|
+
template_manager.render_template(
|
145
|
+
configs.templates.draft_rating_manual_template,
|
146
|
+
{
|
147
|
+
"topic": topic,
|
148
|
+
"criteria": criteria,
|
149
|
+
},
|
150
|
+
)
|
151
|
+
),
|
152
|
+
validator=_validator,
|
153
|
+
system_message=f"# your personal briefing: \n{self.briefing}",
|
154
|
+
**kwargs,
|
155
|
+
)
|
156
|
+
|
157
|
+
async def draft_rating_criteria(
|
158
|
+
self,
|
159
|
+
topic: str,
|
160
|
+
criteria_count: NonNegativeInt = 0,
|
161
|
+
**kwargs: Unpack[ValidateKwargs],
|
162
|
+
) -> Set[str]:
|
163
|
+
"""Drafts rating dimensions based on a topic.
|
164
|
+
|
165
|
+
Args:
|
166
|
+
topic (str): The topic for the rating dimensions.
|
167
|
+
criteria_count (NonNegativeInt, optional): The number of dimensions to draft, 0 means no limit. Defaults to 0.
|
168
|
+
**kwargs (Unpack[ValidateKwargs]): Additional keyword arguments for the LLM usage.
|
169
|
+
|
170
|
+
Returns:
|
171
|
+
Set[str]: A set of rating dimensions.
|
172
|
+
"""
|
173
|
+
|
174
|
+
def _validator(response: str) -> Set[str] | None:
|
175
|
+
if (
|
176
|
+
(json_data := JsonCapture.convert_with(response, orjson.loads)) is not None
|
177
|
+
and isinstance(json_data, list)
|
178
|
+
and all(isinstance(v, str) for v in json_data)
|
179
|
+
and (criteria_count == 0 or len(json_data) == criteria_count)
|
180
|
+
):
|
181
|
+
return set(json_data)
|
182
|
+
return None
|
183
|
+
|
184
|
+
return await self.aask_validate(
|
185
|
+
question=(
|
186
|
+
template_manager.render_template(
|
187
|
+
configs.templates.draft_rating_criteria_template,
|
188
|
+
{
|
189
|
+
"topic": topic,
|
190
|
+
"criteria_count": criteria_count,
|
191
|
+
},
|
192
|
+
)
|
193
|
+
),
|
194
|
+
validator=_validator,
|
195
|
+
system_message=f"# your personal briefing: \n{self.briefing}",
|
196
|
+
**kwargs,
|
197
|
+
)
|
198
|
+
|
199
|
+
async def draft_rating_criteria_from_examples(
|
200
|
+
self,
|
201
|
+
topic: str,
|
202
|
+
examples: List[str],
|
203
|
+
reasons_count: PositiveInt = 2,
|
204
|
+
criteria_count: PositiveInt = 5,
|
205
|
+
**kwargs: Unpack[ValidateKwargs],
|
206
|
+
) -> Set[str]:
|
207
|
+
"""Asynchronously drafts a set of rating criteria based on provided examples.
|
208
|
+
|
209
|
+
This function generates rating criteria by analyzing examples and extracting reasons for comparison,
|
210
|
+
then further condensing these reasons into a specified number of criteria.
|
211
|
+
|
212
|
+
Parameters:
|
213
|
+
topic (str): The subject topic for the rating criteria.
|
214
|
+
examples (List[str]): A list of example texts to analyze.
|
215
|
+
reasons_count (PositiveInt, optional): The number of reasons to extract from each pair of examples. Defaults to 2.
|
216
|
+
criteria_count (PositiveInt, optional): The final number of rating criteria to draft. Defaults to 5.
|
217
|
+
**kwargs (Unpack[ValidateKwargs]): Additional keyword arguments for validation.
|
218
|
+
|
219
|
+
Returns:
|
220
|
+
Set[str]: A set of drafted rating criteria.
|
221
|
+
"""
|
222
|
+
|
223
|
+
def _reasons_validator(response: str) -> List[str] | None:
|
224
|
+
if (
|
225
|
+
(json_data := JsonCapture.convert_with(response, orjson.loads)) is not None
|
226
|
+
and isinstance(json_data, list)
|
227
|
+
and all(isinstance(v, str) for v in json_data)
|
228
|
+
and len(json_data) == reasons_count
|
229
|
+
):
|
230
|
+
return json_data
|
231
|
+
return None
|
232
|
+
|
233
|
+
def _criteria_validator(response: str) -> Set[str] | None:
|
234
|
+
if (
|
235
|
+
(json_data := JsonCapture.convert_with(response, orjson.loads)) is not None
|
236
|
+
and isinstance(json_data, list)
|
237
|
+
and all(isinstance(v, str) for v in json_data)
|
238
|
+
and len(json_data) == criteria_count
|
239
|
+
):
|
240
|
+
return set(json_data)
|
241
|
+
return None
|
242
|
+
|
243
|
+
kwargs = GenerateKwargs(system_message=f"# your personal briefing: \n{self.briefing}", **kwargs)
|
244
|
+
# extract reasons from the comparison of ordered pairs of extracted from examples
|
245
|
+
reasons = flatten(
|
246
|
+
await self.aask_validate_batch(
|
247
|
+
questions=[
|
248
|
+
template_manager.render_template(
|
249
|
+
configs.templates.extract_reasons_from_examples_template,
|
250
|
+
{
|
251
|
+
"topic": topic,
|
252
|
+
"first": pair[0],
|
253
|
+
"second": pair[1],
|
254
|
+
"reasons_count": reasons_count,
|
255
|
+
},
|
256
|
+
)
|
257
|
+
for pair in (permutations(examples, 2))
|
258
|
+
],
|
259
|
+
validator=_reasons_validator,
|
260
|
+
**kwargs,
|
261
|
+
)
|
262
|
+
)
|
263
|
+
# extract certain mount of criteria from reasons according to their importance and frequency
|
264
|
+
return await self.aask_validate(
|
265
|
+
question=(
|
266
|
+
template_manager.render_template(
|
267
|
+
configs.templates.extract_criteria_from_reasons_template,
|
268
|
+
{
|
269
|
+
"topic": topic,
|
270
|
+
"reasons": list(reasons),
|
271
|
+
"criteria_count": criteria_count,
|
272
|
+
},
|
273
|
+
)
|
274
|
+
),
|
275
|
+
validator=_criteria_validator,
|
276
|
+
**kwargs,
|
277
|
+
)
|
@@ -1,7 +1,7 @@
|
|
1
|
-
"""A module for
|
1
|
+
"""A module for the task capabilities of the Fabricatio library."""
|
2
2
|
|
3
3
|
from types import CodeType
|
4
|
-
from typing import Any, Dict, List, Optional,
|
4
|
+
from typing import Any, Dict, List, Optional, Tuple, Unpack
|
5
5
|
|
6
6
|
import orjson
|
7
7
|
from fabricatio._rust_instances import template_manager
|
@@ -129,116 +129,3 @@ class HandleTask(WithBriefing, ToolBoxUsage):
|
|
129
129
|
async def handle(self, task: Task, data: Dict[str, Any], **kwargs: Unpack[ValidateKwargs]) -> Optional[Tuple]:
|
130
130
|
"""Asynchronously handles a task based on a given task object and parameters."""
|
131
131
|
return await self.handle_fin_grind(task, data, **kwargs)
|
132
|
-
|
133
|
-
|
134
|
-
class GiveRating(WithBriefing, LLMUsage):
|
135
|
-
"""A class that provides functionality to rate tasks based on a rating manual and score range."""
|
136
|
-
|
137
|
-
async def rate_fine_grind(
|
138
|
-
self,
|
139
|
-
to_rate: str,
|
140
|
-
rating_manual: Dict[str, str],
|
141
|
-
score_range: Tuple[float, float],
|
142
|
-
**kwargs: Unpack[ValidateKwargs],
|
143
|
-
) -> Dict[str, float]:
|
144
|
-
"""Rates a given task based on a rating manual and score range.
|
145
|
-
|
146
|
-
Args:
|
147
|
-
to_rate: The task to be rated.
|
148
|
-
rating_manual: A dictionary containing the rating criteria.
|
149
|
-
score_range: A tuple representing the valid score range.
|
150
|
-
**kwargs: Additional keyword arguments for the LLM usage.
|
151
|
-
|
152
|
-
Returns:
|
153
|
-
A dictionary with the ratings for each dimension.
|
154
|
-
"""
|
155
|
-
|
156
|
-
def _validator(response: str) -> Dict[str, float] | None:
|
157
|
-
if (
|
158
|
-
(json_data := JsonCapture.convert_with(response, orjson.loads)) is not None
|
159
|
-
and isinstance(json_data, dict)
|
160
|
-
and json_data.keys() == rating_manual.keys()
|
161
|
-
and all(isinstance(v, float) for v in json_data.values())
|
162
|
-
and all(score_range[0] <= v <= score_range[1] for v in json_data.values())
|
163
|
-
):
|
164
|
-
return json_data
|
165
|
-
return None
|
166
|
-
|
167
|
-
return await self.aask_validate(
|
168
|
-
question=(
|
169
|
-
template_manager.render_template(
|
170
|
-
configs.templates.rate_fine_grind_template,
|
171
|
-
{
|
172
|
-
"to_rate": to_rate,
|
173
|
-
"min_score": score_range[0],
|
174
|
-
"max_score": score_range[1],
|
175
|
-
"rating_manual": rating_manual,
|
176
|
-
},
|
177
|
-
)
|
178
|
-
),
|
179
|
-
validator=_validator,
|
180
|
-
system_message=f"# your personal briefing: \n{self.briefing}",
|
181
|
-
**kwargs,
|
182
|
-
)
|
183
|
-
|
184
|
-
async def rate(
|
185
|
-
self,
|
186
|
-
to_rate: str,
|
187
|
-
topic: str,
|
188
|
-
dimensions: Set[str],
|
189
|
-
score_range: Tuple[float, float] = (0.0, 1.0),
|
190
|
-
**kwargs: Unpack[ValidateKwargs],
|
191
|
-
) -> Dict[str, float]:
|
192
|
-
"""Rates a task based on a topic and dimensions. this function will automatically draft a rating manual based on the topic and dimensions.
|
193
|
-
|
194
|
-
Args:
|
195
|
-
to_rate: The task to be rated.
|
196
|
-
topic: The topic related to the task.
|
197
|
-
dimensions: A set of dimensions for rating.
|
198
|
-
score_range: A tuple representing the valid score range
|
199
|
-
**kwargs: Additional keyword arguments for the LLM usage.
|
200
|
-
|
201
|
-
Returns:
|
202
|
-
A dictionary with the ratings for each dimension.
|
203
|
-
"""
|
204
|
-
manual = await self.draft_rating_manual(topic, dimensions, **kwargs)
|
205
|
-
return await self.rate_fine_grind(to_rate, manual, score_range, **kwargs)
|
206
|
-
|
207
|
-
async def draft_rating_manual(
|
208
|
-
self, topic: str, dimensions: Set[str], **kwargs: Unpack[ValidateKwargs]
|
209
|
-
) -> Dict[str, str]:
|
210
|
-
"""Drafts a rating manual based on a topic and dimensions.
|
211
|
-
|
212
|
-
Args:
|
213
|
-
topic: The topic for the rating manual.
|
214
|
-
dimensions: A set of dimensions for the rating manual.
|
215
|
-
**kwargs: Additional keyword arguments for the LLM usage.
|
216
|
-
|
217
|
-
Returns:
|
218
|
-
A dictionary representing the drafted rating manual.
|
219
|
-
"""
|
220
|
-
|
221
|
-
def _validator(response: str) -> Dict[str, str] | None:
|
222
|
-
if (
|
223
|
-
(json_data := JsonCapture.convert_with(response, orjson.loads)) is not None
|
224
|
-
and isinstance(json_data, dict)
|
225
|
-
and json_data.keys() == dimensions
|
226
|
-
and all(isinstance(v, str) for v in json_data.values())
|
227
|
-
):
|
228
|
-
return json_data
|
229
|
-
return None
|
230
|
-
|
231
|
-
return await self.aask_validate(
|
232
|
-
question=(
|
233
|
-
template_manager.render_template(
|
234
|
-
configs.templates.draft_rating_manual_template,
|
235
|
-
{
|
236
|
-
"topic": topic,
|
237
|
-
"dimensions": dimensions,
|
238
|
-
},
|
239
|
-
)
|
240
|
-
),
|
241
|
-
validator=_validator,
|
242
|
-
system_message=f"# your personal briefing: \n{self.briefing}",
|
243
|
-
**kwargs,
|
244
|
-
)
|
@@ -163,6 +163,15 @@ class TemplateConfig(BaseModel):
|
|
163
163
|
draft_rating_manual_template: str = Field(default="draft_rating_manual")
|
164
164
|
"""The name of the draft rating manual template which will be used to draft rating manual."""
|
165
165
|
|
166
|
+
draft_rating_criteria_template: str = Field(default="draft_rating_criteria")
|
167
|
+
"""The name of the draft rating criteria template which will be used to draft rating criteria."""
|
168
|
+
|
169
|
+
extract_reasons_from_examples_template: str = Field(default="extract_reasons_from_examples")
|
170
|
+
"""The name of the extract reasons from examples template which will be used to extract reasons from examples."""
|
171
|
+
|
172
|
+
extract_criteria_from_reasons_template: str = Field(default="extract_criteria_from_reasons")
|
173
|
+
"""The name of the extract criteria from reasons template which will be used to extract criteria from reasons."""
|
174
|
+
|
166
175
|
|
167
176
|
class MagikaConfig(BaseModel):
|
168
177
|
"""Magika configuration class."""
|
@@ -5,15 +5,16 @@ from abc import abstractmethod
|
|
5
5
|
from asyncio import Queue
|
6
6
|
from typing import Any, Dict, Self, Tuple, Type, Union, Unpack
|
7
7
|
|
8
|
+
from fabricatio.capabilities.rating import GiveRating
|
9
|
+
from fabricatio.capabilities.task import HandleTask, ProposeTask
|
8
10
|
from fabricatio.journal import logger
|
9
|
-
from fabricatio.models.advanced import HandleTask, ProposeTask
|
10
11
|
from fabricatio.models.generic import WithBriefing
|
11
12
|
from fabricatio.models.task import Task
|
12
13
|
from fabricatio.models.usages import ToolBoxUsage
|
13
14
|
from pydantic import Field, PrivateAttr
|
14
15
|
|
15
16
|
|
16
|
-
class Action(HandleTask, ProposeTask):
|
17
|
+
class Action(HandleTask, ProposeTask, GiveRating):
|
17
18
|
"""Class that represents an action to be executed in a workflow."""
|
18
19
|
|
19
20
|
personality: str = Field(default="")
|
@@ -24,8 +24,13 @@ class ValidateKwargs(LLMKwargs):
|
|
24
24
|
max_validations: NotRequired[PositiveInt]
|
25
25
|
|
26
26
|
|
27
|
-
class
|
28
|
-
"""A type representing the keyword arguments for the
|
27
|
+
class GenerateKwargs(ValidateKwargs):
|
28
|
+
"""A type representing the keyword arguments for the generate method."""
|
29
29
|
|
30
30
|
system_message: NotRequired[str]
|
31
|
+
|
32
|
+
|
33
|
+
class ChooseKwargs(GenerateKwargs):
|
34
|
+
"""A type representing the keyword arguments for the choose method."""
|
35
|
+
|
31
36
|
k: NotRequired[NonNegativeInt]
|
@@ -2,17 +2,17 @@
|
|
2
2
|
|
3
3
|
from typing import Any, Self, Set
|
4
4
|
|
5
|
+
from fabricatio.capabilities.rating import GiveRating
|
6
|
+
from fabricatio.capabilities.task import HandleTask, ProposeTask
|
5
7
|
from fabricatio.core import env
|
6
8
|
from fabricatio.journal import logger
|
7
9
|
from fabricatio.models.action import WorkFlow
|
8
|
-
from fabricatio.models.advanced import ProposeTask
|
9
10
|
from fabricatio.models.events import Event
|
10
11
|
from fabricatio.models.tool import ToolBox
|
11
|
-
from fabricatio.models.usages import ToolBoxUsage
|
12
12
|
from pydantic import Field
|
13
13
|
|
14
14
|
|
15
|
-
class Role(ProposeTask,
|
15
|
+
class Role(ProposeTask, HandleTask, GiveRating):
|
16
16
|
"""Class that represents a role with a registry of events and workflows."""
|
17
17
|
|
18
18
|
registry: dict[Event | str, WorkFlow] = Field(...)
|