fabricatio 0.3.13__cp312-cp312-manylinux_2_34_x86_64.whl → 0.3.14__cp312-cp312-manylinux_2_34_x86_64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fabricatio/__init__.py +6 -13
- fabricatio/actions/article.py +87 -50
- fabricatio/actions/article_rag.py +59 -68
- fabricatio/actions/output.py +58 -24
- fabricatio/actions/rag.py +2 -3
- fabricatio/capabilities/advanced_judge.py +4 -7
- fabricatio/capabilities/advanced_rag.py +2 -1
- fabricatio/capabilities/censor.py +5 -4
- fabricatio/capabilities/check.py +27 -27
- fabricatio/capabilities/correct.py +22 -22
- fabricatio/capabilities/extract.py +33 -33
- fabricatio/capabilities/persist.py +103 -0
- fabricatio/capabilities/propose.py +2 -2
- fabricatio/capabilities/rag.py +11 -10
- fabricatio/capabilities/rating.py +66 -70
- fabricatio/capabilities/review.py +12 -11
- fabricatio/capabilities/task.py +19 -18
- fabricatio/decorators.py +11 -9
- fabricatio/{core.py → emitter.py} +17 -19
- fabricatio/journal.py +2 -4
- fabricatio/models/action.py +15 -32
- fabricatio/models/extra/aricle_rag.py +13 -8
- fabricatio/models/extra/article_base.py +57 -25
- fabricatio/models/extra/article_essence.py +2 -1
- fabricatio/models/extra/article_main.py +24 -22
- fabricatio/models/extra/article_outline.py +2 -1
- fabricatio/models/extra/article_proposal.py +1 -1
- fabricatio/models/extra/rag.py +2 -2
- fabricatio/models/extra/rule.py +2 -1
- fabricatio/models/generic.py +55 -137
- fabricatio/models/kwargs_types.py +1 -54
- fabricatio/models/role.py +49 -28
- fabricatio/models/task.py +3 -4
- fabricatio/models/tool.py +6 -7
- fabricatio/models/usages.py +146 -149
- fabricatio/parser.py +59 -99
- fabricatio/rust.cpython-312-x86_64-linux-gnu.so +0 -0
- fabricatio/rust.pyi +58 -81
- fabricatio/utils.py +63 -162
- fabricatio-0.3.14.data/scripts/tdown +0 -0
- fabricatio-0.3.14.data/scripts/ttm +0 -0
- {fabricatio-0.3.13.dist-info → fabricatio-0.3.14.dist-info}/METADATA +9 -13
- fabricatio-0.3.14.dist-info/RECORD +64 -0
- {fabricatio-0.3.13.dist-info → fabricatio-0.3.14.dist-info}/WHEEL +1 -1
- fabricatio-0.3.13.data/scripts/tdown +0 -0
- fabricatio-0.3.13.data/scripts/ttm +0 -0
- fabricatio-0.3.13.dist-info/RECORD +0 -63
- {fabricatio-0.3.13.dist-info → fabricatio-0.3.14.dist-info}/licenses/LICENSE +0 -0
fabricatio/__init__.py
CHANGED
@@ -1,36 +1,29 @@
|
|
1
1
|
"""Fabricatio is a Python library for building llm app using event-based agent structure."""
|
2
2
|
|
3
|
-
from fabricatio
|
4
|
-
|
5
|
-
from fabricatio import actions, capabilities, toolboxes, workflows
|
6
|
-
from fabricatio.core import env
|
3
|
+
from fabricatio import actions, capabilities, fs, models, parser, toolboxes, utils, workflows
|
7
4
|
from fabricatio.journal import logger
|
8
|
-
from fabricatio.models import extra
|
9
5
|
from fabricatio.models.action import Action, WorkFlow
|
10
6
|
from fabricatio.models.role import Role
|
11
7
|
from fabricatio.models.task import Task
|
12
8
|
from fabricatio.models.tool import ToolBox
|
13
|
-
from fabricatio.
|
9
|
+
from fabricatio.rust import CONFIG, TEMPLATE_MANAGER, Event
|
14
10
|
|
15
11
|
__all__ = [
|
16
12
|
"CONFIG",
|
17
13
|
"TEMPLATE_MANAGER",
|
18
14
|
"Action",
|
19
|
-
"BibManager",
|
20
|
-
"Capture",
|
21
15
|
"Event",
|
22
|
-
"GenericCapture",
|
23
|
-
"JsonCapture",
|
24
|
-
"PythonCapture",
|
25
16
|
"Role",
|
26
17
|
"Task",
|
27
18
|
"ToolBox",
|
28
19
|
"WorkFlow",
|
29
20
|
"actions",
|
30
21
|
"capabilities",
|
31
|
-
"
|
32
|
-
"extra",
|
22
|
+
"fs",
|
33
23
|
"logger",
|
24
|
+
"models",
|
25
|
+
"parser",
|
34
26
|
"toolboxes",
|
27
|
+
"utils",
|
35
28
|
"workflows",
|
36
29
|
]
|
fabricatio/actions/article.py
CHANGED
@@ -2,9 +2,8 @@
|
|
2
2
|
|
3
3
|
from asyncio import gather
|
4
4
|
from pathlib import Path
|
5
|
-
from typing import Callable, List, Optional
|
5
|
+
from typing import Callable, ClassVar, List, Optional
|
6
6
|
|
7
|
-
from fabricatio.rust import CONFIG, TEMPLATE_MANAGER, BibManager, detect_language
|
8
7
|
from more_itertools import filter_map
|
9
8
|
from pydantic import Field
|
10
9
|
from rich import print as r_print
|
@@ -16,13 +15,14 @@ from fabricatio.fs import dump_text, safe_text_read
|
|
16
15
|
from fabricatio.journal import logger
|
17
16
|
from fabricatio.models.action import Action
|
18
17
|
from fabricatio.models.extra.article_essence import ArticleEssence
|
19
|
-
from fabricatio.models.extra.article_main import Article
|
18
|
+
from fabricatio.models.extra.article_main import Article, ArticleChapter, ArticleSubsection
|
20
19
|
from fabricatio.models.extra.article_outline import ArticleOutline
|
21
20
|
from fabricatio.models.extra.article_proposal import ArticleProposal
|
22
21
|
from fabricatio.models.extra.rule import RuleSet
|
23
22
|
from fabricatio.models.kwargs_types import ValidateKwargs
|
24
23
|
from fabricatio.models.task import Task
|
25
24
|
from fabricatio.models.usages import LLMUsage
|
25
|
+
from fabricatio.rust import CONFIG, TEMPLATE_MANAGER, BibManager, detect_language
|
26
26
|
from fabricatio.utils import ok, wrapp_in_block
|
27
27
|
|
28
28
|
|
@@ -38,10 +38,10 @@ class ExtractArticleEssence(Action, Propose):
|
|
38
38
|
"""The key of the output data."""
|
39
39
|
|
40
40
|
async def _execute(
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
41
|
+
self,
|
42
|
+
task_input: Task,
|
43
|
+
reader: Callable[[str], Optional[str]] = lambda p: Path(p).read_text(encoding="utf-8"),
|
44
|
+
**_,
|
45
45
|
) -> List[ArticleEssence]:
|
46
46
|
if not task_input.dependencies:
|
47
47
|
logger.info(err := "Task not approved, since no dependencies are provided.")
|
@@ -54,11 +54,11 @@ class ExtractArticleEssence(Action, Propose):
|
|
54
54
|
out = []
|
55
55
|
|
56
56
|
for ess in await self.propose(
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
57
|
+
ArticleEssence,
|
58
|
+
[
|
59
|
+
f"{c}\n\n\nBased the provided academic article above, you need to extract the essence from it.\n\nWrite the value string using `{detect_language(c)}`"
|
60
|
+
for c in contents
|
61
|
+
],
|
62
62
|
):
|
63
63
|
if ess is None:
|
64
64
|
logger.warning("Could not extract article essence")
|
@@ -75,10 +75,10 @@ class FixArticleEssence(Action):
|
|
75
75
|
"""The key of the output data."""
|
76
76
|
|
77
77
|
async def _execute(
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
78
|
+
self,
|
79
|
+
bib_mgr: BibManager,
|
80
|
+
article_essence: List[ArticleEssence],
|
81
|
+
**_,
|
82
82
|
) -> List[ArticleEssence]:
|
83
83
|
out = []
|
84
84
|
count = 0
|
@@ -105,11 +105,11 @@ class GenerateArticleProposal(Action, Propose):
|
|
105
105
|
"""The key of the output data."""
|
106
106
|
|
107
107
|
async def _execute(
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
108
|
+
self,
|
109
|
+
task_input: Optional[Task] = None,
|
110
|
+
article_briefing: Optional[str] = None,
|
111
|
+
article_briefing_path: Optional[str] = None,
|
112
|
+
**_,
|
113
113
|
) -> Optional[ArticleProposal]:
|
114
114
|
if article_briefing is None and article_briefing_path is None and task_input is None:
|
115
115
|
logger.error("Task not approved, since all inputs are None.")
|
@@ -148,10 +148,10 @@ class GenerateInitialOutline(Action, Extract):
|
|
148
148
|
"""The kwargs to extract the outline."""
|
149
149
|
|
150
150
|
async def _execute(
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
151
|
+
self,
|
152
|
+
article_proposal: ArticleProposal,
|
153
|
+
supervisor: Optional[bool] = None,
|
154
|
+
**_,
|
155
155
|
) -> Optional[ArticleOutline]:
|
156
156
|
req = (
|
157
157
|
f"Design each chapter of a proper and academic and ready for release manner.\n"
|
@@ -206,10 +206,10 @@ class FixIntrospectedErrors(Action, Censor):
|
|
206
206
|
"""The maximum number of errors to fix."""
|
207
207
|
|
208
208
|
async def _execute(
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
209
|
+
self,
|
210
|
+
article_outline: ArticleOutline,
|
211
|
+
intro_fix_ruleset: Optional[RuleSet] = None,
|
212
|
+
**_,
|
213
213
|
) -> Optional[ArticleOutline]:
|
214
214
|
counter = 0
|
215
215
|
origin = article_outline
|
@@ -241,10 +241,10 @@ class GenerateArticle(Action, Censor):
|
|
241
241
|
ruleset: Optional[RuleSet] = None
|
242
242
|
|
243
243
|
async def _execute(
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
|
244
|
+
self,
|
245
|
+
article_outline: ArticleOutline,
|
246
|
+
article_gen_ruleset: Optional[RuleSet] = None,
|
247
|
+
**_,
|
248
248
|
) -> Optional[Article]:
|
249
249
|
article: Article = Article.from_outline(ok(article_outline, "Article outline not specified.")).update_ref(
|
250
250
|
article_outline
|
@@ -277,43 +277,80 @@ class LoadArticle(Action):
|
|
277
277
|
class WriteChapterSummary(Action, LLMUsage):
|
278
278
|
"""Write the chapter summary."""
|
279
279
|
|
280
|
-
|
280
|
+
ctx_override: ClassVar[bool] = True
|
281
281
|
|
282
282
|
paragraph_count: int = 1
|
283
283
|
|
284
|
-
summary_word_count: int =
|
285
|
-
|
284
|
+
summary_word_count: int = 120
|
285
|
+
output_key: str = "summarized_article"
|
286
286
|
summary_title: str = "Chapter Summary"
|
287
|
-
write_to: Optional[Path] = None
|
288
287
|
|
289
|
-
|
290
|
-
|
288
|
+
skip_chapters: List[str] = Field(default_factory=list)
|
289
|
+
|
290
|
+
async def _execute(self, article_path: Path, **cxt) -> Article:
|
291
|
+
article = Article.from_article_file(article_path, article_path.stem)
|
291
292
|
|
293
|
+
chaps = [c for c in article.chapters if c.title not in self.skip_chapters]
|
294
|
+
|
295
|
+
retained_chapters = []
|
296
|
+
# Count chapters before filtering based on section presence,
|
297
|
+
# chaps at this point has already been filtered by self.skip_chapters
|
298
|
+
initial_chaps_for_summary_step_count = len(chaps)
|
299
|
+
|
300
|
+
for chapter_candidate in chaps:
|
301
|
+
if chapter_candidate.sections: # Check if the sections list is non-empty
|
302
|
+
retained_chapters.append(chapter_candidate)
|
303
|
+
else:
|
304
|
+
# Log c warning for each chapter skipped due to lack of sections
|
305
|
+
logger.warning(
|
306
|
+
f"Chapter '{chapter_candidate.title}' has no sections and will be skipped for summary generation."
|
307
|
+
)
|
308
|
+
|
309
|
+
chaps = retained_chapters # Update chaps to only include chapters with sections
|
310
|
+
|
311
|
+
# If chaps is now empty, but there were chapters to consider at the start of this step,
|
312
|
+
# log c specific warning.
|
313
|
+
if not chaps and initial_chaps_for_summary_step_count > 0:
|
314
|
+
logger.warning(
|
315
|
+
"All chapters considered for summary were skipped as they lack sections. No summaries will be generated for these."
|
316
|
+
)
|
317
|
+
|
318
|
+
# This line was part of the original selection.
|
319
|
+
# It will now log the titles of the chapters that are actually being processed (those with sections).
|
320
|
+
# If 'chaps' is empty, this will result in logger.info(""), which is acceptable.
|
321
|
+
logger.info(";".join(a.title for a in chaps))
|
292
322
|
ret = [
|
293
|
-
|
323
|
+
ArticleSubsection.from_typst_code(self.summary_title, raw)
|
294
324
|
for raw in (
|
295
325
|
await self.aask(
|
296
326
|
TEMPLATE_MANAGER.render_template(
|
297
327
|
CONFIG.templates.chap_summary_template,
|
298
328
|
[
|
299
329
|
{
|
300
|
-
"chapter":
|
301
|
-
"title":
|
302
|
-
"language":
|
330
|
+
"chapter": c.to_typst_code(),
|
331
|
+
"title": c.title,
|
332
|
+
"language": c.language,
|
303
333
|
"summary_word_count": self.summary_word_count,
|
304
334
|
"paragraph_count": self.paragraph_count,
|
305
335
|
}
|
306
|
-
for
|
336
|
+
for c in chaps
|
307
337
|
],
|
308
338
|
)
|
309
339
|
)
|
310
340
|
)
|
311
341
|
]
|
312
342
|
|
313
|
-
|
314
|
-
|
315
|
-
|
316
|
-
|
317
|
-
|
343
|
+
for c, n in zip(chaps, ret, strict=True):
|
344
|
+
c: ArticleChapter
|
345
|
+
n: ArticleSubsection
|
346
|
+
if c.sections[-1].title == self.summary_title:
|
347
|
+
c.sections.pop()
|
348
|
+
|
349
|
+
c.sections[-1].subsections.append(n)
|
318
350
|
|
319
|
-
|
351
|
+
article.update_article_file(article_path)
|
352
|
+
|
353
|
+
article_string = safe_text_read(article_path)
|
354
|
+
article_string.replace(f"=== {self.summary_title}", f"== {self.summary_title}")
|
355
|
+
dump_text(article_path, article_string)
|
356
|
+
return article
|
@@ -4,14 +4,6 @@ from asyncio import gather
|
|
4
4
|
from pathlib import Path
|
5
5
|
from typing import ClassVar, List, Optional
|
6
6
|
|
7
|
-
from fabricatio.rust import (
|
8
|
-
BibManager,
|
9
|
-
convert_all_block_tex,
|
10
|
-
convert_all_inline_tex,
|
11
|
-
convert_to_block_formula,
|
12
|
-
convert_to_inline_formula,
|
13
|
-
fix_misplaced_labels,
|
14
|
-
)
|
15
7
|
from pydantic import Field, PositiveInt
|
16
8
|
|
17
9
|
from fabricatio.capabilities.advanced_rag import AdvancedRAG
|
@@ -27,6 +19,11 @@ from fabricatio.models.extra.article_main import Article, ArticleChapter, Articl
|
|
27
19
|
from fabricatio.models.extra.article_outline import ArticleOutline
|
28
20
|
from fabricatio.models.extra.rule import RuleSet
|
29
21
|
from fabricatio.models.kwargs_types import ChooseKwargs, LLMKwargs
|
22
|
+
from fabricatio.rust import (
|
23
|
+
BibManager,
|
24
|
+
convert_all_tex_math,
|
25
|
+
fix_misplaced_labels,
|
26
|
+
)
|
30
27
|
from fabricatio.utils import ok
|
31
28
|
|
32
29
|
TYPST_CITE_USAGE = (
|
@@ -78,11 +75,11 @@ class WriteArticleContentRAG(Action, Extract, AdvancedRAG):
|
|
78
75
|
tei_endpoint: Optional[str] = None
|
79
76
|
|
80
77
|
async def _execute(
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
78
|
+
self,
|
79
|
+
article_outline: ArticleOutline,
|
80
|
+
collection_name: Optional[str] = None,
|
81
|
+
supervisor: Optional[bool] = None,
|
82
|
+
**cxt,
|
86
83
|
) -> Article:
|
87
84
|
article = Article.from_outline(article_outline).update_ref(article_outline)
|
88
85
|
self.target_collection = collection_name or self.safe_target_collection
|
@@ -103,12 +100,12 @@ class WriteArticleContentRAG(Action, Extract, AdvancedRAG):
|
|
103
100
|
"questionary", "`questionary` is required for supervisor mode, please install it by `fabricatio[qa]`"
|
104
101
|
)
|
105
102
|
async def _supervisor_inner(
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
103
|
+
self,
|
104
|
+
article: Article,
|
105
|
+
article_outline: ArticleOutline,
|
106
|
+
chap: ArticleChapter,
|
107
|
+
sec: ArticleSection,
|
108
|
+
subsec: ArticleSubsection,
|
112
109
|
) -> ArticleSubsection:
|
113
110
|
from questionary import confirm, text
|
114
111
|
from rich import print as r_print
|
@@ -129,20 +126,19 @@ class WriteArticleContentRAG(Action, Extract, AdvancedRAG):
|
|
129
126
|
raw_paras = edt
|
130
127
|
|
131
128
|
raw_paras = fix_misplaced_labels(raw_paras)
|
132
|
-
raw_paras =
|
133
|
-
raw_paras = convert_all_block_tex(raw_paras)
|
129
|
+
raw_paras = convert_all_tex_math(raw_paras)
|
134
130
|
|
135
131
|
r_print(raw_paras)
|
136
132
|
|
137
133
|
return await self.extract_new_subsec(subsec, raw_paras, cm)
|
138
134
|
|
139
135
|
async def _inner(
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
136
|
+
self,
|
137
|
+
article: Article,
|
138
|
+
article_outline: ArticleOutline,
|
139
|
+
chap: ArticleChapter,
|
140
|
+
sec: ArticleSection,
|
141
|
+
subsec: ArticleSubsection,
|
146
142
|
) -> ArticleSubsection:
|
147
143
|
cm = CitationManager()
|
148
144
|
|
@@ -153,13 +149,12 @@ class WriteArticleContentRAG(Action, Extract, AdvancedRAG):
|
|
153
149
|
raw_paras = "\n".join(p for p in raw_paras.splitlines() if p and not p.endswith("**") and not p.startswith("#"))
|
154
150
|
|
155
151
|
raw_paras = fix_misplaced_labels(raw_paras)
|
156
|
-
raw_paras =
|
157
|
-
raw_paras = convert_all_block_tex(raw_paras)
|
152
|
+
raw_paras = convert_all_tex_math(raw_paras)
|
158
153
|
|
159
154
|
return await self.extract_new_subsec(subsec, raw_paras, cm)
|
160
155
|
|
161
156
|
async def extract_new_subsec(
|
162
|
-
|
157
|
+
self, subsec: ArticleSubsection, raw_paras: str, cm: CitationManager
|
163
158
|
) -> ArticleSubsection:
|
164
159
|
"""Extract the new subsec."""
|
165
160
|
new_subsec = ok(
|
@@ -182,14 +177,14 @@ class WriteArticleContentRAG(Action, Extract, AdvancedRAG):
|
|
182
177
|
return subsec
|
183
178
|
|
184
179
|
async def write_raw(
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
180
|
+
self,
|
181
|
+
article: Article,
|
182
|
+
article_outline: ArticleOutline,
|
183
|
+
chap: ArticleChapter,
|
184
|
+
sec: ArticleSection,
|
185
|
+
subsec: ArticleSubsection,
|
186
|
+
cm: CitationManager,
|
187
|
+
extra_instruction: str = "",
|
193
188
|
) -> str:
|
194
189
|
"""Write the raw paragraphs of the subsec."""
|
195
190
|
return await self.aask(
|
@@ -205,14 +200,14 @@ class WriteArticleContentRAG(Action, Extract, AdvancedRAG):
|
|
205
200
|
)
|
206
201
|
|
207
202
|
async def search_database(
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
203
|
+
self,
|
204
|
+
article: Article,
|
205
|
+
article_outline: ArticleOutline,
|
206
|
+
chap: ArticleChapter,
|
207
|
+
sec: ArticleSection,
|
208
|
+
subsec: ArticleSubsection,
|
209
|
+
cm: CitationManager,
|
210
|
+
extra_instruction: str = "",
|
216
211
|
) -> None:
|
217
212
|
"""Search database for related references."""
|
218
213
|
search_req = (
|
@@ -261,8 +256,6 @@ class ArticleConsultRAG(Action, AdvancedRAG):
|
|
261
256
|
from questionary import confirm, text
|
262
257
|
from rich import print as r_print
|
263
258
|
|
264
|
-
from fabricatio.rust import convert_all_block_tex, convert_all_inline_tex, fix_misplaced_labels
|
265
|
-
|
266
259
|
self.target_collection = collection_name or self.safe_target_collection
|
267
260
|
|
268
261
|
cm = CitationManager()
|
@@ -272,8 +265,7 @@ class ArticleConsultRAG(Action, AdvancedRAG):
|
|
272
265
|
if await confirm("Empty the cm?").ask_async():
|
273
266
|
cm.empty()
|
274
267
|
|
275
|
-
req =
|
276
|
-
req = convert_to_inline_formula(req)
|
268
|
+
req = convert_all_tex_math(req)
|
277
269
|
|
278
270
|
await self.clued_search(
|
279
271
|
req,
|
@@ -289,8 +281,7 @@ class ArticleConsultRAG(Action, AdvancedRAG):
|
|
289
281
|
ret = await self.aask(f"{cm.as_prompt()}\n{self.req}\n{req}")
|
290
282
|
|
291
283
|
ret = fix_misplaced_labels(ret)
|
292
|
-
ret =
|
293
|
-
ret = convert_all_block_tex(ret)
|
284
|
+
ret = convert_all_tex_math(ret)
|
294
285
|
ret = cm.apply(ret)
|
295
286
|
|
296
287
|
r_print(ret)
|
@@ -321,12 +312,12 @@ class TweakArticleRAG(Action, RAG, Censor):
|
|
321
312
|
"""The limit of references to be retrieved"""
|
322
313
|
|
323
314
|
async def _execute(
|
324
|
-
|
325
|
-
|
326
|
-
|
327
|
-
|
328
|
-
|
329
|
-
|
315
|
+
self,
|
316
|
+
article: Article,
|
317
|
+
collection_name: str = "article_essence",
|
318
|
+
twk_rag_ruleset: Optional[RuleSet] = None,
|
319
|
+
parallel: bool = False,
|
320
|
+
**cxt,
|
330
321
|
) -> Article:
|
331
322
|
"""Write an article based on the provided outline.
|
332
323
|
|
@@ -381,10 +372,10 @@ class TweakArticleRAG(Action, RAG, Censor):
|
|
381
372
|
subsec,
|
382
373
|
ruleset=ruleset,
|
383
374
|
reference=f"{'\n\n'.join(d.display() for d in await self.aretrieve(refind_q, document_model=ArticleEssence, max_accepted=self.ref_limit))}\n\n"
|
384
|
-
|
385
|
-
|
386
|
-
|
387
|
-
|
375
|
+
f"You can use Reference above to rewrite the `{subsec.__class__.__name__}`.\n"
|
376
|
+
f"You should Always use `{subsec.language}` as written language, "
|
377
|
+
f"which is the original language of the `{subsec.title}`. "
|
378
|
+
f"since rewrite a `{subsec.__class__.__name__}` in a different language is usually a bad choice",
|
388
379
|
)
|
389
380
|
|
390
381
|
|
@@ -399,12 +390,12 @@ class ChunkArticle(Action):
|
|
399
390
|
"""The maximum overlapping rate between chunks."""
|
400
391
|
|
401
392
|
async def _execute(
|
402
|
-
|
403
|
-
|
404
|
-
|
405
|
-
|
406
|
-
|
407
|
-
|
393
|
+
self,
|
394
|
+
article_path: str | Path,
|
395
|
+
bib_manager: BibManager,
|
396
|
+
max_chunk_size: Optional[int] = None,
|
397
|
+
max_overlapping_rate: Optional[float] = None,
|
398
|
+
**_,
|
408
399
|
) -> List[ArticleChunk]:
|
409
400
|
return ArticleChunk.from_file(
|
410
401
|
article_path,
|
fabricatio/actions/output.py
CHANGED
@@ -1,16 +1,16 @@
|
|
1
1
|
"""Dump the finalized output to a file."""
|
2
2
|
|
3
3
|
from pathlib import Path
|
4
|
-
from typing import Any, Iterable, List, Mapping, Optional, Type
|
5
|
-
|
6
|
-
from fabricatio.rust import TEMPLATE_MANAGER
|
4
|
+
from typing import Any, Iterable, List, Mapping, Optional, Self, Sequence, Type
|
7
5
|
|
6
|
+
from fabricatio.capabilities.persist import PersistentAble
|
8
7
|
from fabricatio.fs import dump_text
|
9
8
|
from fabricatio.journal import logger
|
10
9
|
from fabricatio.models.action import Action
|
11
|
-
from fabricatio.models.generic import FinalizedDumpAble, FromMapping,
|
10
|
+
from fabricatio.models.generic import FinalizedDumpAble, FromMapping, FromSequence
|
12
11
|
from fabricatio.models.task import Task
|
13
12
|
from fabricatio.models.usages import LLMUsage
|
13
|
+
from fabricatio.rust import TEMPLATE_MANAGER
|
14
14
|
from fabricatio.utils import ok
|
15
15
|
|
16
16
|
|
@@ -21,11 +21,11 @@ class DumpFinalizedOutput(Action, LLMUsage):
|
|
21
21
|
dump_path: Optional[str] = None
|
22
22
|
|
23
23
|
async def _execute(
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
24
|
+
self,
|
25
|
+
to_dump: FinalizedDumpAble,
|
26
|
+
task_input: Optional[Task] = None,
|
27
|
+
dump_path: Optional[str | Path] = None,
|
28
|
+
**_,
|
29
29
|
) -> str:
|
30
30
|
dump_path = Path(
|
31
31
|
dump_path
|
@@ -52,11 +52,11 @@ class RenderedDump(Action, LLMUsage):
|
|
52
52
|
"""The template name to render the data."""
|
53
53
|
|
54
54
|
async def _execute(
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
55
|
+
self,
|
56
|
+
to_dump: FinalizedDumpAble,
|
57
|
+
task_input: Optional[Task] = None,
|
58
|
+
dump_path: Optional[str | Path] = None,
|
59
|
+
**_,
|
60
60
|
) -> str:
|
61
61
|
dump_path = Path(
|
62
62
|
dump_path
|
@@ -91,10 +91,10 @@ class PersistentAll(Action, LLMUsage):
|
|
91
91
|
"""Whether to remove the existing dir before dumping."""
|
92
92
|
|
93
93
|
async def _execute(
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
94
|
+
self,
|
95
|
+
task_input: Optional[Task] = None,
|
96
|
+
persist_dir: Optional[str | Path] = None,
|
97
|
+
**cxt,
|
98
98
|
) -> int:
|
99
99
|
persist_dir = Path(
|
100
100
|
persist_dir
|
@@ -124,7 +124,7 @@ class PersistentAll(Action, LLMUsage):
|
|
124
124
|
v.persist(final_dir)
|
125
125
|
count += 1
|
126
126
|
if isinstance(v, Iterable) and any(
|
127
|
-
|
127
|
+
persistent_ables := (pers for pers in v if isinstance(pers, PersistentAble))
|
128
128
|
):
|
129
129
|
logger.info(f"Persisting collection {k} to {final_dir}")
|
130
130
|
final_dir.mkdir(parents=True, exist_ok=True)
|
@@ -174,11 +174,11 @@ class RetrieveFromLatest[T: PersistentAble](RetrieveFromPersistent[T], FromMappi
|
|
174
174
|
|
175
175
|
@classmethod
|
176
176
|
def from_mapping(
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
177
|
+
cls,
|
178
|
+
mapping: Mapping[str, str | Path],
|
179
|
+
*,
|
180
|
+
retrieve_cls: Type[T],
|
181
|
+
**kwargs,
|
182
182
|
) -> List["RetrieveFromLatest[T]"]:
|
183
183
|
"""Create a list of `RetrieveFromLatest` from the mapping."""
|
184
184
|
return [
|
@@ -212,3 +212,37 @@ class GatherAsList(Action):
|
|
212
212
|
result = [cxt[k] for k in cxt if k.startswith(self.gather_prefix)]
|
213
213
|
logger.debug(f"Gathered {len(result)} items with prefix {self.gather_prefix}")
|
214
214
|
return result
|
215
|
+
|
216
|
+
|
217
|
+
class Forward(Action, FromMapping, FromSequence):
|
218
|
+
"""Forward the object from the context to the output."""
|
219
|
+
|
220
|
+
output_key: str = "forwarded"
|
221
|
+
"""Gather the objects from the context as a list."""
|
222
|
+
original: str
|
223
|
+
|
224
|
+
async def _execute(self, *_: Any, **cxt) -> Any:
|
225
|
+
source = cxt.get(self.original)
|
226
|
+
if source is None:
|
227
|
+
logger.warning(f"Original object {self.original} not found in the context")
|
228
|
+
return source
|
229
|
+
|
230
|
+
@classmethod
|
231
|
+
def from_sequence(cls, sequence: Sequence[str], *, original: str, **kwargs: Any) -> List[Self]:
|
232
|
+
"""Create a list of `Forward` from the sequence."""
|
233
|
+
return [cls(original=original, output_key=o, **kwargs) for o in sequence]
|
234
|
+
|
235
|
+
@classmethod
|
236
|
+
def from_mapping(cls, mapping: Mapping[str, str | Sequence[str]], **kwargs: Any) -> List[Self]:
|
237
|
+
"""Create a list of `Forward` from the mapping."""
|
238
|
+
actions = []
|
239
|
+
for original_key, output_val in mapping.items():
|
240
|
+
if isinstance(output_val, str):
|
241
|
+
actions.append(cls(original=original_key, output_key=output_val, **kwargs))
|
242
|
+
elif isinstance(output_val, Sequence):
|
243
|
+
actions.extend(cls(original=original_key, output_key=output_key, **kwargs) for output_key in output_val)
|
244
|
+
else:
|
245
|
+
logger.warning(
|
246
|
+
f"Invalid type for output key value in mapping: {type(output_val)} for original key {original_key}. Expected str or Sequence[str]."
|
247
|
+
)
|
248
|
+
return actions
|
fabricatio/actions/rag.py
CHANGED
@@ -2,13 +2,12 @@
|
|
2
2
|
|
3
3
|
from typing import List, Optional
|
4
4
|
|
5
|
-
from fabricatio.rust import CONFIG
|
6
|
-
|
7
5
|
from fabricatio.capabilities.rag import RAG
|
8
6
|
from fabricatio.journal import logger
|
9
7
|
from fabricatio.models.action import Action
|
10
8
|
from fabricatio.models.extra.rag import MilvusClassicModel, MilvusDataBase
|
11
9
|
from fabricatio.models.task import Task
|
10
|
+
from fabricatio.rust import CONFIG
|
12
11
|
from fabricatio.utils import ok
|
13
12
|
|
14
13
|
|
@@ -20,7 +19,7 @@ class InjectToDB(Action, RAG):
|
|
20
19
|
"""The name of the collection to inject data into."""
|
21
20
|
|
22
21
|
async def _execute[T: MilvusDataBase](
|
23
|
-
|
22
|
+
self, to_inject: Optional[T] | List[Optional[T]], override_inject: bool = False, **_
|
24
23
|
) -> Optional[str]:
|
25
24
|
from pymilvus.milvus_client import IndexParams
|
26
25
|
|