fabricatio 0.2.1.dev0__cp313-cp313-win_amd64.whl → 0.3.14.dev4__cp313-cp313-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fabricatio/__init__.py +12 -20
- fabricatio/actions/__init__.py +1 -5
- fabricatio/actions/article.py +319 -0
- fabricatio/actions/article_rag.py +416 -0
- fabricatio/actions/fs.py +25 -0
- fabricatio/actions/output.py +248 -0
- fabricatio/actions/rag.py +96 -0
- fabricatio/actions/rules.py +83 -0
- fabricatio/capabilities/__init__.py +1 -0
- fabricatio/capabilities/advanced_judge.py +20 -0
- fabricatio/capabilities/advanced_rag.py +61 -0
- fabricatio/capabilities/censor.py +105 -0
- fabricatio/capabilities/check.py +212 -0
- fabricatio/capabilities/correct.py +228 -0
- fabricatio/capabilities/extract.py +74 -0
- fabricatio/capabilities/persist.py +103 -0
- fabricatio/capabilities/propose.py +65 -0
- fabricatio/capabilities/rag.py +263 -0
- fabricatio/capabilities/rating.py +404 -0
- fabricatio/capabilities/review.py +114 -0
- fabricatio/capabilities/task.py +113 -0
- fabricatio/decorators.py +251 -179
- fabricatio/{core.py → emitter.py} +31 -21
- fabricatio/fs/__init__.py +32 -2
- fabricatio/fs/curd.py +32 -9
- fabricatio/fs/readers.py +44 -7
- fabricatio/journal.py +3 -19
- fabricatio/models/action.py +185 -61
- fabricatio/models/adv_kwargs_types.py +63 -0
- fabricatio/models/extra/__init__.py +1 -0
- fabricatio/models/extra/advanced_judge.py +32 -0
- fabricatio/models/extra/aricle_rag.py +284 -0
- fabricatio/models/extra/article_base.py +422 -0
- fabricatio/models/extra/article_essence.py +101 -0
- fabricatio/models/extra/article_main.py +285 -0
- fabricatio/models/extra/article_outline.py +46 -0
- fabricatio/models/extra/article_proposal.py +52 -0
- fabricatio/models/extra/patches.py +20 -0
- fabricatio/models/extra/problem.py +165 -0
- fabricatio/models/extra/rag.py +98 -0
- fabricatio/models/extra/rule.py +52 -0
- fabricatio/models/generic.py +704 -36
- fabricatio/models/kwargs_types.py +112 -17
- fabricatio/models/role.py +74 -27
- fabricatio/models/task.py +94 -60
- fabricatio/models/tool.py +328 -188
- fabricatio/models/usages.py +791 -515
- fabricatio/parser.py +81 -60
- fabricatio/rust.cp313-win_amd64.pyd +0 -0
- fabricatio/rust.pyi +846 -0
- fabricatio/toolboxes/__init__.py +1 -3
- fabricatio/toolboxes/fs.py +17 -1
- fabricatio/utils.py +156 -0
- fabricatio/workflows/__init__.py +1 -0
- fabricatio/workflows/articles.py +24 -0
- fabricatio/workflows/rag.py +11 -0
- fabricatio-0.3.14.dev4.data/scripts/tdown.exe +0 -0
- fabricatio-0.3.14.dev4.data/scripts/ttm.exe +0 -0
- fabricatio-0.3.14.dev4.dist-info/METADATA +188 -0
- fabricatio-0.3.14.dev4.dist-info/RECORD +64 -0
- {fabricatio-0.2.1.dev0.dist-info → fabricatio-0.3.14.dev4.dist-info}/WHEEL +1 -1
- fabricatio/_rust.cp313-win_amd64.pyd +0 -0
- fabricatio/_rust.pyi +0 -53
- fabricatio/_rust_instances.py +0 -8
- fabricatio/actions/communication.py +0 -15
- fabricatio/actions/transmission.py +0 -23
- fabricatio/config.py +0 -263
- fabricatio/models/advanced.py +0 -128
- fabricatio/models/events.py +0 -82
- fabricatio/models/utils.py +0 -78
- fabricatio/toolboxes/task.py +0 -6
- fabricatio-0.2.1.dev0.data/scripts/tdown.exe +0 -0
- fabricatio-0.2.1.dev0.dist-info/METADATA +0 -420
- fabricatio-0.2.1.dev0.dist-info/RECORD +0 -35
- {fabricatio-0.2.1.dev0.dist-info → fabricatio-0.3.14.dev4.dist-info}/licenses/LICENSE +0 -0
@@ -0,0 +1,416 @@
|
|
1
|
+
"""A module for writing articles using RAG (Retrieval-Augmented Generation) capabilities."""
|
2
|
+
|
3
|
+
from asyncio import gather
|
4
|
+
from pathlib import Path
|
5
|
+
from typing import ClassVar, List, Optional
|
6
|
+
|
7
|
+
from pydantic import Field, PositiveInt
|
8
|
+
|
9
|
+
from fabricatio.capabilities.advanced_rag import AdvancedRAG
|
10
|
+
from fabricatio.capabilities.censor import Censor
|
11
|
+
from fabricatio.capabilities.extract import Extract
|
12
|
+
from fabricatio.capabilities.rag import RAG
|
13
|
+
from fabricatio.decorators import precheck_package
|
14
|
+
from fabricatio.journal import logger
|
15
|
+
from fabricatio.models.action import Action
|
16
|
+
from fabricatio.models.extra.aricle_rag import ArticleChunk, CitationManager
|
17
|
+
from fabricatio.models.extra.article_essence import ArticleEssence
|
18
|
+
from fabricatio.models.extra.article_main import Article, ArticleChapter, ArticleSection, ArticleSubsection
|
19
|
+
from fabricatio.models.extra.article_outline import ArticleOutline
|
20
|
+
from fabricatio.models.extra.rule import RuleSet
|
21
|
+
from fabricatio.models.kwargs_types import ChooseKwargs, LLMKwargs
|
22
|
+
from fabricatio.rust import (
|
23
|
+
BibManager,
|
24
|
+
convert_all_block_tex,
|
25
|
+
convert_all_inline_tex,
|
26
|
+
convert_to_block_formula,
|
27
|
+
convert_to_inline_formula,
|
28
|
+
fix_misplaced_labels,
|
29
|
+
)
|
30
|
+
from fabricatio.utils import ok
|
31
|
+
|
32
|
+
TYPST_CITE_USAGE = (
|
33
|
+
"citation number is REQUIRED to cite any reference!'\n"
|
34
|
+
"Legal citing syntax examples(seperated by |): [[1]]|[[1,2]]|[[1-3]]|[[12,13-15]]|[[1-3,5-7]]\n"
|
35
|
+
"Illegal citing syntax examples(seperated by |): [[1],[2],[3]]|[[1],[1-2]]\n"
|
36
|
+
"You SHALL not cite a single reference more than once!"
|
37
|
+
"It's recommended to cite multiple references that supports your conclusion at a time.\n"
|
38
|
+
)
|
39
|
+
|
40
|
+
TYPST_MATH_USAGE = (
|
41
|
+
"Wrap inline expression with '\\(' and '\\)',like '\\(>5m\\)' '\\(89%\\)', and wrap block equation with '\\[' and '\\]'.\n"
|
42
|
+
"In addition to that, you can add a label outside the block equation which can be used as a cross reference identifier, the label is a string wrapped in `<` and `>` like `<energy-release-rate-equation>`.Note that the label string should be a summarizing title for the equation being labeled and should never be written within the formula block.\n"
|
43
|
+
"you can refer to that label by using the syntax with prefix of `@eqt:`, which indicate that this notation is citing a label from the equations. For example ' @eqt:energy-release-rate-equation ' DO remember that the notation shall have both suffixed and prefixed space char which enable the compiler to distinguish the notation from the plaintext."
|
44
|
+
"Below is two usage example:\n"
|
45
|
+
"```typst\n"
|
46
|
+
"See @eqt:mass-energy-equation , it's the foundation of physics.\n"
|
47
|
+
"\\[\n"
|
48
|
+
"E = m c^2\n"
|
49
|
+
"\\] <mass-energy-equation>\n\n\n"
|
50
|
+
"In @eqt:mass-energy-equation , \\(m\\) stands for mass, \\(c\\) stands for speed of light, and \\(E\\) stands for energy. \n"
|
51
|
+
"```\n"
|
52
|
+
)
|
53
|
+
|
54
|
+
|
55
|
+
class WriteArticleContentRAG(Action, Extract, AdvancedRAG):
|
56
|
+
"""Write an article based on the provided outline."""
|
57
|
+
|
58
|
+
ctx_override: ClassVar[bool] = True
|
59
|
+
search_increment_multiplier: float = 1.6
|
60
|
+
"""The increment multiplier of the search increment."""
|
61
|
+
ref_limit: int = 35
|
62
|
+
"""The limit of references to be retrieved"""
|
63
|
+
threshold: float = 0.62
|
64
|
+
"""The threshold of relevance"""
|
65
|
+
extractor_model: LLMKwargs
|
66
|
+
"""The model to use for extracting the content from the retrieved references."""
|
67
|
+
query_model: ChooseKwargs
|
68
|
+
"""The model to use for querying the database"""
|
69
|
+
supervisor: bool = False
|
70
|
+
"""Whether to use supervisor mode"""
|
71
|
+
result_per_query: PositiveInt = 4
|
72
|
+
"""The number of results to be returned per query."""
|
73
|
+
cite_req: str = TYPST_CITE_USAGE
|
74
|
+
"""The req of the write article content."""
|
75
|
+
|
76
|
+
math_req: str = TYPST_MATH_USAGE
|
77
|
+
"""The req of the write article content."""
|
78
|
+
tei_endpoint: Optional[str] = None
|
79
|
+
|
80
|
+
async def _execute(
|
81
|
+
self,
|
82
|
+
article_outline: ArticleOutline,
|
83
|
+
collection_name: Optional[str] = None,
|
84
|
+
supervisor: Optional[bool] = None,
|
85
|
+
**cxt,
|
86
|
+
) -> Article:
|
87
|
+
article = Article.from_outline(article_outline).update_ref(article_outline)
|
88
|
+
self.target_collection = collection_name or self.safe_target_collection
|
89
|
+
if supervisor or (supervisor is None and self.supervisor):
|
90
|
+
for chap, sec, subsec in article.iter_subsections():
|
91
|
+
await self._supervisor_inner(article, article_outline, chap, sec, subsec)
|
92
|
+
|
93
|
+
else:
|
94
|
+
await gather(
|
95
|
+
*[
|
96
|
+
self._inner(article, article_outline, chap, sec, subsec)
|
97
|
+
for chap, sec, subsec in article.iter_subsections()
|
98
|
+
]
|
99
|
+
)
|
100
|
+
return article.convert_tex()
|
101
|
+
|
102
|
+
@precheck_package(
|
103
|
+
"questionary", "`questionary` is required for supervisor mode, please install it by `fabricatio[qa]`"
|
104
|
+
)
|
105
|
+
async def _supervisor_inner(
|
106
|
+
self,
|
107
|
+
article: Article,
|
108
|
+
article_outline: ArticleOutline,
|
109
|
+
chap: ArticleChapter,
|
110
|
+
sec: ArticleSection,
|
111
|
+
subsec: ArticleSubsection,
|
112
|
+
) -> ArticleSubsection:
|
113
|
+
from questionary import confirm, text
|
114
|
+
from rich import print as r_print
|
115
|
+
|
116
|
+
cm = CitationManager()
|
117
|
+
await self.search_database(article, article_outline, chap, sec, subsec, cm)
|
118
|
+
|
119
|
+
raw_paras = await self.write_raw(article, article_outline, chap, sec, subsec, cm)
|
120
|
+
r_print(raw_paras)
|
121
|
+
|
122
|
+
while not await confirm("Accept this version and continue?").ask_async():
|
123
|
+
if inst := await text("Search for more refs for additional spec.").ask_async():
|
124
|
+
await self.search_database(article, article_outline, chap, sec, subsec, cm, extra_instruction=inst)
|
125
|
+
|
126
|
+
if instruction := await text("Enter the instructions to improve").ask_async():
|
127
|
+
raw_paras = await self.write_raw(article, article_outline, chap, sec, subsec, cm, instruction)
|
128
|
+
if edt := await text("Edit", default=raw_paras).ask_async():
|
129
|
+
raw_paras = edt
|
130
|
+
|
131
|
+
raw_paras = fix_misplaced_labels(raw_paras)
|
132
|
+
raw_paras = convert_all_inline_tex(raw_paras)
|
133
|
+
raw_paras = convert_all_block_tex(raw_paras)
|
134
|
+
|
135
|
+
r_print(raw_paras)
|
136
|
+
|
137
|
+
return await self.extract_new_subsec(subsec, raw_paras, cm)
|
138
|
+
|
139
|
+
async def _inner(
|
140
|
+
self,
|
141
|
+
article: Article,
|
142
|
+
article_outline: ArticleOutline,
|
143
|
+
chap: ArticleChapter,
|
144
|
+
sec: ArticleSection,
|
145
|
+
subsec: ArticleSubsection,
|
146
|
+
) -> ArticleSubsection:
|
147
|
+
cm = CitationManager()
|
148
|
+
|
149
|
+
await self.search_database(article, article_outline, chap, sec, subsec, cm)
|
150
|
+
|
151
|
+
raw_paras = await self.write_raw(article, article_outline, chap, sec, subsec, cm)
|
152
|
+
|
153
|
+
raw_paras = "\n".join(p for p in raw_paras.splitlines() if p and not p.endswith("**") and not p.startswith("#"))
|
154
|
+
|
155
|
+
raw_paras = fix_misplaced_labels(raw_paras)
|
156
|
+
raw_paras = convert_all_inline_tex(raw_paras)
|
157
|
+
raw_paras = convert_all_block_tex(raw_paras)
|
158
|
+
|
159
|
+
return await self.extract_new_subsec(subsec, raw_paras, cm)
|
160
|
+
|
161
|
+
async def extract_new_subsec(
|
162
|
+
self, subsec: ArticleSubsection, raw_paras: str, cm: CitationManager
|
163
|
+
) -> ArticleSubsection:
|
164
|
+
"""Extract the new subsec."""
|
165
|
+
new_subsec = ok(
|
166
|
+
await self.extract(
|
167
|
+
ArticleSubsection,
|
168
|
+
raw_paras,
|
169
|
+
f"Above is the subsection titled `{subsec.title}`.\n"
|
170
|
+
f"I need you to extract the content to construct a new `{ArticleSubsection.__class__.__name__}`,"
|
171
|
+
f"Do not attempt to change the original content, your job is ONLY content extraction",
|
172
|
+
**self.extractor_model,
|
173
|
+
),
|
174
|
+
"Failed to propose new subsection.",
|
175
|
+
)
|
176
|
+
|
177
|
+
for p in new_subsec.paragraphs:
|
178
|
+
p.content = cm.apply(p.content)
|
179
|
+
p.description = cm.apply(p.description)
|
180
|
+
subsec.update_from(new_subsec)
|
181
|
+
logger.debug(f"{subsec.title}:rpl\n{subsec.display()}")
|
182
|
+
return subsec
|
183
|
+
|
184
|
+
async def write_raw(
|
185
|
+
self,
|
186
|
+
article: Article,
|
187
|
+
article_outline: ArticleOutline,
|
188
|
+
chap: ArticleChapter,
|
189
|
+
sec: ArticleSection,
|
190
|
+
subsec: ArticleSubsection,
|
191
|
+
cm: CitationManager,
|
192
|
+
extra_instruction: str = "",
|
193
|
+
) -> str:
|
194
|
+
"""Write the raw paragraphs of the subsec."""
|
195
|
+
return await self.aask(
|
196
|
+
f"{cm.as_prompt()}\nAbove is some related reference from other auther retrieved for you."
|
197
|
+
f"{article_outline.finalized_dump()}\n\nAbove is my article outline, I m writing graduate thesis titled `{article.title}`. "
|
198
|
+
f"More specifically, i m witting the Chapter `{chap.title}` >> Section `{sec.title}` >> Subsection `{subsec.title}`.\n"
|
199
|
+
f"Please help me write the paragraphs of the subsec mentioned above, which is `{subsec.title}`.\n"
|
200
|
+
f"{self.cite_req}\n{self.math_req}\n"
|
201
|
+
f"You SHALL use `{article.language}` as writing language.\n{extra_instruction}\n"
|
202
|
+
f"Do not use numbered list to display the outcome, you should regard you are writing the main text of the thesis.\n"
|
203
|
+
f"You should not copy others' works from the references directly on to my thesis, we can only harness the conclusion they have drawn.\n"
|
204
|
+
f"No extra explanation is allowed."
|
205
|
+
)
|
206
|
+
|
207
|
+
async def search_database(
|
208
|
+
self,
|
209
|
+
article: Article,
|
210
|
+
article_outline: ArticleOutline,
|
211
|
+
chap: ArticleChapter,
|
212
|
+
sec: ArticleSection,
|
213
|
+
subsec: ArticleSubsection,
|
214
|
+
cm: CitationManager,
|
215
|
+
extra_instruction: str = "",
|
216
|
+
) -> None:
|
217
|
+
"""Search database for related references."""
|
218
|
+
search_req = (
|
219
|
+
f"{article_outline.finalized_dump()}\n\nAbove is my article outline, I m writing graduate thesis titled `{article.title}`. "
|
220
|
+
f"More specifically, i m witting the Chapter `{chap.title}` >> Section `{sec.title}` >> Subsection `{subsec.title}`.\n"
|
221
|
+
f"I need to search related references to build up the content of the subsec mentioned above, which is `{subsec.title}`.\n"
|
222
|
+
f"provide 10~16 queries as possible, to get best result!\n"
|
223
|
+
f"You should provide both English version and chinese version of the refined queries!\n{extra_instruction}"
|
224
|
+
)
|
225
|
+
|
226
|
+
await self.clued_search(
|
227
|
+
search_req,
|
228
|
+
cm,
|
229
|
+
refinery_kwargs=self.query_model,
|
230
|
+
expand_multiplier=self.search_increment_multiplier,
|
231
|
+
base_accepted=self.ref_limit,
|
232
|
+
result_per_query=self.result_per_query,
|
233
|
+
similarity_threshold=self.threshold,
|
234
|
+
tei_endpoint=self.tei_endpoint,
|
235
|
+
)
|
236
|
+
|
237
|
+
|
238
|
+
class ArticleConsultRAG(Action, AdvancedRAG):
|
239
|
+
"""Write an article based on the provided outline."""
|
240
|
+
|
241
|
+
ctx_override: ClassVar[bool] = True
|
242
|
+
output_key: str = "consult_count"
|
243
|
+
search_increment_multiplier: float = 1.6
|
244
|
+
"""The multiplier to increase the limit of references to retrieve per query."""
|
245
|
+
ref_limit: int = 26
|
246
|
+
"""The final limit of references."""
|
247
|
+
ref_per_q: int = 13
|
248
|
+
"""The limit of references to retrieve per query."""
|
249
|
+
similarity_threshold: float = 0.62
|
250
|
+
"""The similarity threshold of references to retrieve."""
|
251
|
+
ref_q_model: ChooseKwargs = Field(default_factory=ChooseKwargs)
|
252
|
+
"""The model to use for refining query."""
|
253
|
+
req: str = TYPST_CITE_USAGE
|
254
|
+
"""The request for the rag model."""
|
255
|
+
tei_endpoint: Optional[str] = None
|
256
|
+
|
257
|
+
@precheck_package(
|
258
|
+
"questionary", "`questionary` is required for supervisor mode, please install it by `fabricatio[qa]`"
|
259
|
+
)
|
260
|
+
async def _execute(self, collection_name: Optional[str] = None, **cxt) -> int:
|
261
|
+
from questionary import confirm, text
|
262
|
+
from rich import print as r_print
|
263
|
+
|
264
|
+
from fabricatio.rust import convert_all_block_tex, convert_all_inline_tex, fix_misplaced_labels
|
265
|
+
|
266
|
+
self.target_collection = collection_name or self.safe_target_collection
|
267
|
+
|
268
|
+
cm = CitationManager()
|
269
|
+
|
270
|
+
counter = 0
|
271
|
+
while (req := await text("User: ").ask_async()) is not None:
|
272
|
+
if await confirm("Empty the cm?").ask_async():
|
273
|
+
cm.empty()
|
274
|
+
|
275
|
+
req = convert_to_block_formula(req)
|
276
|
+
req = convert_to_inline_formula(req)
|
277
|
+
|
278
|
+
await self.clued_search(
|
279
|
+
req,
|
280
|
+
cm,
|
281
|
+
refinery_kwargs=self.ref_q_model,
|
282
|
+
expand_multiplier=self.search_increment_multiplier,
|
283
|
+
base_accepted=self.ref_limit,
|
284
|
+
result_per_query=self.ref_per_q,
|
285
|
+
similarity_threshold=self.similarity_threshold,
|
286
|
+
tei_endpoint=self.tei_endpoint,
|
287
|
+
)
|
288
|
+
|
289
|
+
ret = await self.aask(f"{cm.as_prompt()}\n{self.req}\n{req}")
|
290
|
+
|
291
|
+
ret = fix_misplaced_labels(ret)
|
292
|
+
ret = convert_all_inline_tex(ret)
|
293
|
+
ret = convert_all_block_tex(ret)
|
294
|
+
ret = cm.apply(ret)
|
295
|
+
|
296
|
+
r_print(ret)
|
297
|
+
counter += 1
|
298
|
+
logger.info(f"{counter} rounds of conversation.")
|
299
|
+
return counter
|
300
|
+
|
301
|
+
|
302
|
+
class TweakArticleRAG(Action, RAG, Censor):
|
303
|
+
"""Write an article based on the provided outline.
|
304
|
+
|
305
|
+
This class inherits from `Action`, `RAG`, and `Censor` to provide capabilities for writing and refining articles
|
306
|
+
using Retrieval-Augmented Generation (RAG) techniques. It processes an article outline, enhances subsections by
|
307
|
+
searching for related references, and applies censoring rules to ensure compliance with the provided ruleset.
|
308
|
+
|
309
|
+
Attributes:
|
310
|
+
output_key (str): The key used to store the output of the action.
|
311
|
+
ruleset (Optional[RuleSet]): The ruleset to be used for censoring the article.
|
312
|
+
"""
|
313
|
+
|
314
|
+
output_key: str = "rag_tweaked_article"
|
315
|
+
"""The key used to store the output of the action."""
|
316
|
+
|
317
|
+
ruleset: Optional[RuleSet] = None
|
318
|
+
"""The ruleset to be used for censoring the article."""
|
319
|
+
|
320
|
+
ref_limit: int = 30
|
321
|
+
"""The limit of references to be retrieved"""
|
322
|
+
|
323
|
+
async def _execute(
|
324
|
+
self,
|
325
|
+
article: Article,
|
326
|
+
collection_name: str = "article_essence",
|
327
|
+
twk_rag_ruleset: Optional[RuleSet] = None,
|
328
|
+
parallel: bool = False,
|
329
|
+
**cxt,
|
330
|
+
) -> Article:
|
331
|
+
"""Write an article based on the provided outline.
|
332
|
+
|
333
|
+
This method processes the article outline, either in parallel or sequentially, by enhancing each subsection
|
334
|
+
with relevant references and applying censoring rules.
|
335
|
+
|
336
|
+
Args:
|
337
|
+
article (Article): The article to be processed.
|
338
|
+
collection_name (str): The name of the collection to view for processing.
|
339
|
+
twk_rag_ruleset (Optional[RuleSet]): The ruleset to apply for censoring. If not provided, the class's ruleset is used.
|
340
|
+
parallel (bool): If True, process subsections in parallel. Otherwise, process them sequentially.
|
341
|
+
**cxt: Additional context parameters.
|
342
|
+
|
343
|
+
Returns:
|
344
|
+
Article: The processed article with enhanced subsections and applied censoring rules.
|
345
|
+
"""
|
346
|
+
self.view(collection_name)
|
347
|
+
|
348
|
+
if parallel:
|
349
|
+
await gather(
|
350
|
+
*[
|
351
|
+
self._inner(article, subsec, ok(twk_rag_ruleset or self.ruleset, "No ruleset provided!"))
|
352
|
+
for _, __, subsec in article.iter_subsections()
|
353
|
+
],
|
354
|
+
return_exceptions=True,
|
355
|
+
)
|
356
|
+
else:
|
357
|
+
for _, __, subsec in article.iter_subsections():
|
358
|
+
await self._inner(article, subsec, ok(twk_rag_ruleset or self.ruleset, "No ruleset provided!"))
|
359
|
+
return article
|
360
|
+
|
361
|
+
async def _inner(self, article: Article, subsec: ArticleSubsection, ruleset: RuleSet) -> None:
|
362
|
+
"""Enhance a subsection of the article with references and apply censoring rules.
|
363
|
+
|
364
|
+
This method refines the query for the subsection, retrieves related references, and applies censoring rules
|
365
|
+
to the subsection's paragraphs.
|
366
|
+
|
367
|
+
Args:
|
368
|
+
article (Article): The article containing the subsection.
|
369
|
+
subsec (ArticleSubsection): The subsection to be enhanced.
|
370
|
+
ruleset (RuleSet): The ruleset to apply for censoring.
|
371
|
+
|
372
|
+
Returns:
|
373
|
+
None
|
374
|
+
"""
|
375
|
+
refind_q = ok(
|
376
|
+
await self.arefined_query(
|
377
|
+
f"{article.referenced.as_prompt()}\n# Subsection requiring reference enhancement\n{subsec.display()}\n"
|
378
|
+
)
|
379
|
+
)
|
380
|
+
await self.censor_obj_inplace(
|
381
|
+
subsec,
|
382
|
+
ruleset=ruleset,
|
383
|
+
reference=f"{'\n\n'.join(d.display() for d in await self.aretrieve(refind_q, document_model=ArticleEssence, max_accepted=self.ref_limit))}\n\n"
|
384
|
+
f"You can use Reference above to rewrite the `{subsec.__class__.__name__}`.\n"
|
385
|
+
f"You should Always use `{subsec.language}` as written language, "
|
386
|
+
f"which is the original language of the `{subsec.title}`. "
|
387
|
+
f"since rewrite a `{subsec.__class__.__name__}` in a different language is usually a bad choice",
|
388
|
+
)
|
389
|
+
|
390
|
+
|
391
|
+
class ChunkArticle(Action):
|
392
|
+
"""Chunk an article into smaller chunks."""
|
393
|
+
|
394
|
+
output_key: str = "article_chunks"
|
395
|
+
"""The key used to store the output of the action."""
|
396
|
+
max_chunk_size: Optional[int] = None
|
397
|
+
"""The maximum size of each chunk."""
|
398
|
+
max_overlapping_rate: Optional[float] = None
|
399
|
+
"""The maximum overlapping rate between chunks."""
|
400
|
+
|
401
|
+
async def _execute(
|
402
|
+
self,
|
403
|
+
article_path: str | Path,
|
404
|
+
bib_manager: BibManager,
|
405
|
+
max_chunk_size: Optional[int] = None,
|
406
|
+
max_overlapping_rate: Optional[float] = None,
|
407
|
+
**_,
|
408
|
+
) -> List[ArticleChunk]:
|
409
|
+
return ArticleChunk.from_file(
|
410
|
+
article_path,
|
411
|
+
bib_manager,
|
412
|
+
max_chunk_size=ok(max_chunk_size or self.max_chunk_size, "No max_chunk_size provided!"),
|
413
|
+
max_overlapping_rate=ok(
|
414
|
+
max_overlapping_rate or self.max_overlapping_rate, "No max_overlapping_rate provided!"
|
415
|
+
),
|
416
|
+
)
|
fabricatio/actions/fs.py
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
"""A module for file system utilities."""
|
2
|
+
|
3
|
+
from pathlib import Path
|
4
|
+
from typing import Any, List, Mapping, Self
|
5
|
+
|
6
|
+
from fabricatio.fs import safe_text_read
|
7
|
+
from fabricatio.journal import logger
|
8
|
+
from fabricatio.models.action import Action
|
9
|
+
from fabricatio.models.generic import FromMapping
|
10
|
+
|
11
|
+
|
12
|
+
class ReadText(Action, FromMapping):
|
13
|
+
"""Read text from a file."""
|
14
|
+
output_key: str = "read_text"
|
15
|
+
read_path: str | Path
|
16
|
+
"""Path to the file to read."""
|
17
|
+
|
18
|
+
async def _execute(self, *_: Any, **cxt) -> str:
|
19
|
+
logger.info(f"Read text from {Path(self.read_path).as_posix()} to {self.output_key}")
|
20
|
+
return safe_text_read(self.read_path)
|
21
|
+
|
22
|
+
@classmethod
|
23
|
+
def from_mapping(cls, mapping: Mapping[str, str | Path], **kwargs: Any) -> List[Self]:
|
24
|
+
"""Create a list of ReadText actions from a mapping of output_key to read_path."""
|
25
|
+
return [cls(read_path=p, output_key=k, **kwargs) for k, p in mapping.items()]
|
@@ -0,0 +1,248 @@
|
|
1
|
+
"""Dump the finalized output to a file."""
|
2
|
+
|
3
|
+
from pathlib import Path
|
4
|
+
from typing import Any, Iterable, List, Mapping, Optional, Self, Sequence, Type
|
5
|
+
|
6
|
+
from fabricatio.capabilities.persist import PersistentAble
|
7
|
+
from fabricatio.fs import dump_text
|
8
|
+
from fabricatio.journal import logger
|
9
|
+
from fabricatio.models.action import Action
|
10
|
+
from fabricatio.models.generic import FinalizedDumpAble, FromMapping, FromSequence
|
11
|
+
from fabricatio.models.task import Task
|
12
|
+
from fabricatio.models.usages import LLMUsage
|
13
|
+
from fabricatio.rust import TEMPLATE_MANAGER
|
14
|
+
from fabricatio.utils import ok
|
15
|
+
|
16
|
+
|
17
|
+
class DumpFinalizedOutput(Action, LLMUsage):
|
18
|
+
"""Dump the finalized output to a file."""
|
19
|
+
|
20
|
+
output_key: str = "dump_path"
|
21
|
+
dump_path: Optional[str] = None
|
22
|
+
|
23
|
+
async def _execute(
|
24
|
+
self,
|
25
|
+
to_dump: FinalizedDumpAble,
|
26
|
+
task_input: Optional[Task] = None,
|
27
|
+
dump_path: Optional[str | Path] = None,
|
28
|
+
**_,
|
29
|
+
) -> str:
|
30
|
+
dump_path = Path(
|
31
|
+
dump_path
|
32
|
+
or self.dump_path
|
33
|
+
or ok(
|
34
|
+
await self.awhich_pathstr(
|
35
|
+
f"{ok(task_input, 'Neither `task_input` and `dump_path` is provided.').briefing}\n\nExtract a single path of the file, to which I will dump the data."
|
36
|
+
),
|
37
|
+
"Could not find the path of file to dump the data.",
|
38
|
+
)
|
39
|
+
)
|
40
|
+
logger.info(f"Saving output to {dump_path.as_posix()}")
|
41
|
+
ok(to_dump, "Could not dump the data since the path is not specified.").finalized_dump_to(dump_path)
|
42
|
+
return dump_path.as_posix()
|
43
|
+
|
44
|
+
|
45
|
+
class RenderedDump(Action, LLMUsage):
|
46
|
+
"""Render the data to a file."""
|
47
|
+
|
48
|
+
output_key: str = "dump_path"
|
49
|
+
dump_path: Optional[str] = None
|
50
|
+
|
51
|
+
template_name: str
|
52
|
+
"""The template name to render the data."""
|
53
|
+
|
54
|
+
async def _execute(
|
55
|
+
self,
|
56
|
+
to_dump: FinalizedDumpAble,
|
57
|
+
task_input: Optional[Task] = None,
|
58
|
+
dump_path: Optional[str | Path] = None,
|
59
|
+
**_,
|
60
|
+
) -> str:
|
61
|
+
dump_path = Path(
|
62
|
+
dump_path
|
63
|
+
or self.dump_path
|
64
|
+
or ok(
|
65
|
+
await self.awhich_pathstr(
|
66
|
+
f"{ok(task_input, 'Neither `task_input` and `dump_path` is provided.').briefing}\n\nExtract a single path of the file, to which I will dump the data."
|
67
|
+
),
|
68
|
+
"Could not find the path of file to dump the data.",
|
69
|
+
)
|
70
|
+
)
|
71
|
+
|
72
|
+
logger.info(f"Saving output to {dump_path.as_posix()}")
|
73
|
+
dump_text(
|
74
|
+
dump_path,
|
75
|
+
TEMPLATE_MANAGER.render_template(
|
76
|
+
self.template_name, {to_dump.__class__.__name__: to_dump.finalized_dump()}
|
77
|
+
),
|
78
|
+
)
|
79
|
+
|
80
|
+
return dump_path.as_posix()
|
81
|
+
|
82
|
+
|
83
|
+
class PersistentAll(Action, LLMUsage):
|
84
|
+
"""Persist all the data to a file."""
|
85
|
+
|
86
|
+
output_key: str = "persistent_count"
|
87
|
+
"""The number of objects persisted."""
|
88
|
+
persist_dir: Optional[str] = None
|
89
|
+
"""The directory to persist the data."""
|
90
|
+
override: bool = False
|
91
|
+
"""Whether to remove the existing dir before dumping."""
|
92
|
+
|
93
|
+
async def _execute(
|
94
|
+
self,
|
95
|
+
task_input: Optional[Task] = None,
|
96
|
+
persist_dir: Optional[str | Path] = None,
|
97
|
+
**cxt,
|
98
|
+
) -> int:
|
99
|
+
persist_dir = Path(
|
100
|
+
persist_dir
|
101
|
+
or self.persist_dir
|
102
|
+
or ok(
|
103
|
+
await self.awhich_pathstr(
|
104
|
+
f"{ok(task_input, 'Neither `task_input` and `dump_path` is provided.').briefing}\n\nExtract a single path of the file, to which I will persist the data."
|
105
|
+
),
|
106
|
+
"Can not find the path of file to persist the data.",
|
107
|
+
)
|
108
|
+
)
|
109
|
+
|
110
|
+
count = 0
|
111
|
+
if persist_dir.is_file():
|
112
|
+
logger.warning("Dump should be a directory, but it is a file. Skip dumping.")
|
113
|
+
return count
|
114
|
+
if self.override and persist_dir.is_dir():
|
115
|
+
logger.info(f"Override the existing directory {persist_dir.as_posix()}.")
|
116
|
+
persist_dir.rmdir()
|
117
|
+
logger.info(f"Starting persistence in directory {persist_dir}")
|
118
|
+
for k, v in cxt.items():
|
119
|
+
final_dir = persist_dir.joinpath(k)
|
120
|
+
logger.debug(f"Checking key {k} for persistence")
|
121
|
+
if isinstance(v, PersistentAble):
|
122
|
+
logger.info(f"Persisting object {k} to {final_dir}")
|
123
|
+
final_dir.mkdir(parents=True, exist_ok=True)
|
124
|
+
v.persist(final_dir)
|
125
|
+
count += 1
|
126
|
+
if isinstance(v, Iterable) and any(
|
127
|
+
persistent_ables := (pers for pers in v if isinstance(pers, PersistentAble))
|
128
|
+
):
|
129
|
+
logger.info(f"Persisting collection {k} to {final_dir}")
|
130
|
+
final_dir.mkdir(parents=True, exist_ok=True)
|
131
|
+
for per in persistent_ables:
|
132
|
+
per.persist(final_dir)
|
133
|
+
count += 1
|
134
|
+
logger.info(f"Persisted {count} objects to {persist_dir}")
|
135
|
+
return count
|
136
|
+
|
137
|
+
|
138
|
+
class RetrieveFromPersistent[T: PersistentAble](Action):
|
139
|
+
"""Retrieve the object from the persistent file."""
|
140
|
+
|
141
|
+
output_key: str = "retrieved_obj"
|
142
|
+
"""Retrieve the object from the persistent file."""
|
143
|
+
load_path: str
|
144
|
+
"""The path of the persistent file or directory contains multiple file."""
|
145
|
+
retrieve_cls: Type[T]
|
146
|
+
"""The class of the object to retrieve."""
|
147
|
+
|
148
|
+
async def _execute(self, /, **_) -> Optional[T | List[T]]:
|
149
|
+
logger.info(f"Retrieve `{self.retrieve_cls.__name__}` from {self.load_path}")
|
150
|
+
if not (p := Path(self.load_path)).exists():
|
151
|
+
logger.warning(f"Path {self.load_path} does not exist")
|
152
|
+
return None
|
153
|
+
|
154
|
+
if p.is_dir():
|
155
|
+
logger.info(f"Found directory with {len(list(p.glob('*')))} items")
|
156
|
+
return [self.retrieve_cls.from_persistent(per) for per in p.glob("*")]
|
157
|
+
return self.retrieve_cls.from_persistent(self.load_path)
|
158
|
+
|
159
|
+
|
160
|
+
class RetrieveFromLatest[T: PersistentAble](RetrieveFromPersistent[T], FromMapping):
|
161
|
+
"""Retrieve the object from the latest persistent file in the dir at `load_path`."""
|
162
|
+
|
163
|
+
async def _execute(self, /, **_) -> Optional[T]:
|
164
|
+
logger.info(f"Retrieve latest `{self.retrieve_cls.__name__}` from {self.load_path}")
|
165
|
+
if not (p := Path(self.load_path)).exists():
|
166
|
+
logger.warning(f"Path {self.load_path} does not exist")
|
167
|
+
return None
|
168
|
+
|
169
|
+
if p.is_dir():
|
170
|
+
logger.info(f"Found directory with {len(list(p.glob('*')))} items")
|
171
|
+
return self.retrieve_cls.from_latest_persistent(self.load_path)
|
172
|
+
logger.error(f"Path {self.load_path} is not a directory")
|
173
|
+
return None
|
174
|
+
|
175
|
+
@classmethod
|
176
|
+
def from_mapping(
|
177
|
+
cls,
|
178
|
+
mapping: Mapping[str, str | Path],
|
179
|
+
*,
|
180
|
+
retrieve_cls: Type[T],
|
181
|
+
**kwargs,
|
182
|
+
) -> List["RetrieveFromLatest[T]"]:
|
183
|
+
"""Create a list of `RetrieveFromLatest` from the mapping."""
|
184
|
+
return [
|
185
|
+
cls(retrieve_cls=retrieve_cls, load_path=Path(p).as_posix(), output_key=o, **kwargs)
|
186
|
+
for o, p in mapping.items()
|
187
|
+
]
|
188
|
+
|
189
|
+
|
190
|
+
class GatherAsList(Action):
|
191
|
+
"""Gather the objects from the context as a list.
|
192
|
+
|
193
|
+
Notes:
|
194
|
+
If both `gather_suffix` and `gather_prefix` are specified, only the objects with the suffix will be gathered.
|
195
|
+
"""
|
196
|
+
|
197
|
+
output_key: str = "gathered"
|
198
|
+
"""Gather the objects from the context as a list."""
|
199
|
+
gather_suffix: Optional[str] = None
|
200
|
+
"""Gather the objects from the context as a list."""
|
201
|
+
gather_prefix: Optional[str] = None
|
202
|
+
"""Gather the objects from the context as a list."""
|
203
|
+
|
204
|
+
async def _execute(self, **cxt) -> List[Any]:
|
205
|
+
if self.gather_suffix is not None:
|
206
|
+
result = [cxt[k] for k in cxt if k.endswith(self.gather_suffix)]
|
207
|
+
logger.debug(f"Gathered {len(result)} items with suffix {self.gather_suffix}")
|
208
|
+
return result
|
209
|
+
if self.gather_prefix is None:
|
210
|
+
logger.error(err := "Either `gather_suffix` or `gather_prefix` must be specified.")
|
211
|
+
raise ValueError(err)
|
212
|
+
result = [cxt[k] for k in cxt if k.startswith(self.gather_prefix)]
|
213
|
+
logger.debug(f"Gathered {len(result)} items with prefix {self.gather_prefix}")
|
214
|
+
return result
|
215
|
+
|
216
|
+
|
217
|
+
class Forward(Action, FromMapping, FromSequence):
|
218
|
+
"""Forward the object from the context to the output."""
|
219
|
+
|
220
|
+
output_key: str = "forwarded"
|
221
|
+
"""Gather the objects from the context as a list."""
|
222
|
+
original: str
|
223
|
+
|
224
|
+
async def _execute(self, *_: Any, **cxt) -> Any:
|
225
|
+
source = cxt.get(self.original)
|
226
|
+
if source is None:
|
227
|
+
logger.warning(f"Original object {self.original} not found in the context")
|
228
|
+
return source
|
229
|
+
|
230
|
+
@classmethod
|
231
|
+
def from_sequence(cls, sequence: Sequence[str], *, original: str, **kwargs: Any) -> List[Self]:
|
232
|
+
"""Create a list of `Forward` from the sequence."""
|
233
|
+
return [cls(original=original, output_key=o, **kwargs) for o in sequence]
|
234
|
+
|
235
|
+
@classmethod
|
236
|
+
def from_mapping(cls, mapping: Mapping[str, str | Sequence[str]], **kwargs: Any) -> List[Self]:
|
237
|
+
"""Create a list of `Forward` from the mapping."""
|
238
|
+
actions = []
|
239
|
+
for original_key, output_val in mapping.items():
|
240
|
+
if isinstance(output_val, str):
|
241
|
+
actions.append(cls(original=original_key, output_key=output_val, **kwargs))
|
242
|
+
elif isinstance(output_val, Sequence):
|
243
|
+
actions.extend(cls(original=original_key, output_key=output_key, **kwargs) for output_key in output_val)
|
244
|
+
else:
|
245
|
+
logger.warning(
|
246
|
+
f"Invalid type for output key value in mapping: {type(output_val)} for original key {original_key}. Expected str or Sequence[str]."
|
247
|
+
)
|
248
|
+
return actions
|