fabricatio 0.2.4.dev2__cp312-cp312-win_amd64.whl → 0.2.5__cp312-cp312-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fabricatio/__init__.py +14 -5
- fabricatio/_rust.cp312-win_amd64.pyd +0 -0
- fabricatio/_rust.pyi +65 -16
- fabricatio/_rust_instances.py +2 -0
- fabricatio/actions/article.py +46 -14
- fabricatio/actions/output.py +21 -0
- fabricatio/actions/rag.py +1 -1
- fabricatio/capabilities/propose.py +14 -20
- fabricatio/capabilities/rag.py +85 -26
- fabricatio/capabilities/rating.py +59 -51
- fabricatio/capabilities/review.py +241 -0
- fabricatio/capabilities/task.py +7 -8
- fabricatio/config.py +36 -4
- fabricatio/fs/__init__.py +13 -1
- fabricatio/fs/curd.py +27 -8
- fabricatio/fs/readers.py +6 -3
- fabricatio/journal.py +1 -1
- fabricatio/models/action.py +6 -8
- fabricatio/models/events.py +6 -4
- fabricatio/models/extra.py +100 -25
- fabricatio/models/generic.py +56 -4
- fabricatio/models/kwargs_types.py +123 -35
- fabricatio/models/role.py +3 -3
- fabricatio/models/task.py +0 -14
- fabricatio/models/tool.py +7 -6
- fabricatio/models/usages.py +144 -101
- fabricatio/parser.py +26 -5
- fabricatio/toolboxes/__init__.py +1 -3
- fabricatio/toolboxes/fs.py +17 -1
- fabricatio/workflows/articles.py +10 -6
- fabricatio/workflows/rag.py +11 -0
- fabricatio-0.2.5.data/scripts/tdown.exe +0 -0
- {fabricatio-0.2.4.dev2.dist-info → fabricatio-0.2.5.dist-info}/METADATA +2 -1
- fabricatio-0.2.5.dist-info/RECORD +41 -0
- fabricatio/toolboxes/task.py +0 -6
- fabricatio-0.2.4.dev2.data/scripts/tdown.exe +0 -0
- fabricatio-0.2.4.dev2.dist-info/RECORD +0 -39
- {fabricatio-0.2.4.dev2.dist-info → fabricatio-0.2.5.dist-info}/WHEEL +0 -0
- {fabricatio-0.2.4.dev2.dist-info → fabricatio-0.2.5.dist-info}/licenses/LICENSE +0 -0
fabricatio/models/usages.py
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
"""This module contains classes that manage the usage of language models and tools in tasks."""
|
2
2
|
|
3
3
|
from asyncio import gather
|
4
|
-
from typing import Callable, Dict, Iterable, List, Optional, Self, Set, Type, Union, Unpack, overload
|
4
|
+
from typing import Callable, Dict, Iterable, List, Optional, Self, Sequence, Set, Type, Union, Unpack, overload
|
5
5
|
|
6
6
|
import asyncstdlib
|
7
7
|
import litellm
|
@@ -9,7 +9,7 @@ from fabricatio._rust_instances import template_manager
|
|
9
9
|
from fabricatio.config import configs
|
10
10
|
from fabricatio.journal import logger
|
11
11
|
from fabricatio.models.generic import ScopedConfig, WithBriefing
|
12
|
-
from fabricatio.models.kwargs_types import ChooseKwargs, EmbeddingKwargs, GenerateKwargs, LLMKwargs
|
12
|
+
from fabricatio.models.kwargs_types import ChooseKwargs, EmbeddingKwargs, GenerateKwargs, LLMKwargs, ValidateKwargs
|
13
13
|
from fabricatio.models.task import Task
|
14
14
|
from fabricatio.models.tool import Tool, ToolBox
|
15
15
|
from fabricatio.models.utils import Messages
|
@@ -20,11 +20,16 @@ from litellm.types.utils import (
|
|
20
20
|
EmbeddingResponse,
|
21
21
|
ModelResponse,
|
22
22
|
StreamingChoices,
|
23
|
+
TextChoices,
|
23
24
|
)
|
24
|
-
from litellm.utils import CustomStreamWrapper
|
25
|
+
from litellm.utils import CustomStreamWrapper # pyright: ignore [reportPrivateImportUsage]
|
25
26
|
from more_itertools import duplicates_everseen
|
26
27
|
from pydantic import Field, NonNegativeInt, PositiveInt
|
27
28
|
|
29
|
+
if configs.cache.enabled and configs.cache.type:
|
30
|
+
litellm.enable_cache(type=configs.cache.type, **configs.cache.params)
|
31
|
+
logger.success(f"{configs.cache.type.name} Cache enabled")
|
32
|
+
|
28
33
|
|
29
34
|
class LLMUsage(ScopedConfig):
|
30
35
|
"""Class that manages LLM (Large Language Model) usage parameters and methods."""
|
@@ -63,6 +68,12 @@ class LLMUsage(ScopedConfig):
|
|
63
68
|
max_retries=kwargs.get("max_retries") or self.llm_max_retries or configs.llm.max_retries,
|
64
69
|
api_key=(self.llm_api_key or configs.llm.api_key).get_secret_value(),
|
65
70
|
base_url=(self.llm_api_endpoint or configs.llm.api_endpoint).unicode_string(),
|
71
|
+
cache={
|
72
|
+
"no-cache": kwargs.get("no_cache"),
|
73
|
+
"no-store": kwargs.get("no_store"),
|
74
|
+
"cache-ttl": kwargs.get("cache_ttl"),
|
75
|
+
"s-maxage": kwargs.get("s_maxage"),
|
76
|
+
},
|
66
77
|
)
|
67
78
|
|
68
79
|
async def ainvoke(
|
@@ -71,7 +82,7 @@ class LLMUsage(ScopedConfig):
|
|
71
82
|
system_message: str = "",
|
72
83
|
n: PositiveInt | None = None,
|
73
84
|
**kwargs: Unpack[LLMKwargs],
|
74
|
-
) ->
|
85
|
+
) -> Sequence[TextChoices | Choices | StreamingChoices]:
|
75
86
|
"""Asynchronously invokes the language model with a question and optional system message.
|
76
87
|
|
77
88
|
Args:
|
@@ -91,13 +102,14 @@ class LLMUsage(ScopedConfig):
|
|
91
102
|
if isinstance(resp, ModelResponse):
|
92
103
|
return resp.choices
|
93
104
|
if isinstance(resp, CustomStreamWrapper):
|
94
|
-
if not configs.debug.streaming_visible:
|
95
|
-
return
|
105
|
+
if not configs.debug.streaming_visible and (pack := stream_chunk_builder(await asyncstdlib.list())):
|
106
|
+
return pack.choices
|
96
107
|
chunks = []
|
97
108
|
async for chunk in resp:
|
98
109
|
chunks.append(chunk)
|
99
110
|
print(chunk.choices[0].delta.content or "", end="") # noqa: T201
|
100
|
-
|
111
|
+
if pack := stream_chunk_builder(chunks):
|
112
|
+
return pack.choices
|
101
113
|
logger.critical(err := f"Unexpected response type: {type(resp)}")
|
102
114
|
raise ValueError(err)
|
103
115
|
|
@@ -105,14 +117,14 @@ class LLMUsage(ScopedConfig):
|
|
105
117
|
async def aask(
|
106
118
|
self,
|
107
119
|
question: List[str],
|
108
|
-
system_message:
|
120
|
+
system_message: List[str],
|
109
121
|
**kwargs: Unpack[LLMKwargs],
|
110
122
|
) -> List[str]: ...
|
111
123
|
@overload
|
112
124
|
async def aask(
|
113
125
|
self,
|
114
126
|
question: str,
|
115
|
-
system_message:
|
127
|
+
system_message: List[str],
|
116
128
|
**kwargs: Unpack[LLMKwargs],
|
117
129
|
) -> List[str]: ...
|
118
130
|
@overload
|
@@ -148,103 +160,115 @@ class LLMUsage(ScopedConfig):
|
|
148
160
|
str | List[str]: The content of the model's response message.
|
149
161
|
"""
|
150
162
|
system_message = system_message or ""
|
151
|
-
match (
|
152
|
-
case (
|
163
|
+
match (question, system_message):
|
164
|
+
case (list(q_seq), list(sm_seq)):
|
153
165
|
res = await gather(
|
154
166
|
*[
|
155
167
|
self.ainvoke(n=1, question=q, system_message=sm, **kwargs)
|
156
|
-
for q, sm in zip(
|
168
|
+
for q, sm in zip(q_seq, sm_seq, strict=True)
|
157
169
|
]
|
158
170
|
)
|
159
|
-
return [r.
|
160
|
-
case (
|
161
|
-
res = await gather(
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
)
|
169
|
-
return [r.pop().message.content for r in res]
|
170
|
-
case (False, False):
|
171
|
-
return (
|
172
|
-
(
|
173
|
-
await self.ainvoke(
|
174
|
-
n=1,
|
175
|
-
question=question,
|
176
|
-
system_message=system_message,
|
177
|
-
**kwargs,
|
178
|
-
)
|
179
|
-
).pop()
|
180
|
-
).message.content
|
171
|
+
return [r[0].message.content for r in res]
|
172
|
+
case (list(q_seq), str(sm)):
|
173
|
+
res = await gather(*[self.ainvoke(n=1, question=q, system_message=sm, **kwargs) for q in q_seq])
|
174
|
+
return [r[0].message.content for r in res]
|
175
|
+
case (str(q), list(sm_seq)):
|
176
|
+
res = await gather(*[self.ainvoke(n=1, question=q, system_message=sm, **kwargs) for sm in sm_seq])
|
177
|
+
return [r[0].message.content for r in res]
|
178
|
+
case (str(q), str(sm)):
|
179
|
+
return ((await self.ainvoke(n=1, question=q, system_message=sm, **kwargs))[0]).message.content
|
181
180
|
case _:
|
182
181
|
raise RuntimeError("Should not reach here.")
|
183
182
|
|
183
|
+
@overload
|
184
184
|
async def aask_validate[T](
|
185
185
|
self,
|
186
186
|
question: str,
|
187
187
|
validator: Callable[[str], T | None],
|
188
|
+
default: T,
|
188
189
|
max_validations: PositiveInt = 2,
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
190
|
+
**kwargs: Unpack[GenerateKwargs],
|
191
|
+
) -> T: ...
|
192
|
+
@overload
|
193
|
+
async def aask_validate[T](
|
194
|
+
self,
|
195
|
+
question: List[str],
|
196
|
+
validator: Callable[[str], T | None],
|
197
|
+
default: T,
|
198
|
+
max_validations: PositiveInt = 2,
|
199
|
+
**kwargs: Unpack[GenerateKwargs],
|
200
|
+
) -> List[T]: ...
|
201
|
+
@overload
|
202
|
+
async def aask_validate[T](
|
203
|
+
self,
|
204
|
+
question: str,
|
205
|
+
validator: Callable[[str], T | None],
|
206
|
+
default: None=None,
|
207
|
+
max_validations: PositiveInt = 2,
|
208
|
+
**kwargs: Unpack[GenerateKwargs],
|
209
|
+
) -> Optional[T]: ...
|
203
210
|
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
)
|
214
|
-
) and (validated := validator(response)):
|
215
|
-
logger.debug(f"Successfully validated the response at {i}th attempt.")
|
216
|
-
return validated
|
217
|
-
logger.error(err := f"Failed to validate the response after {max_validations} attempts.")
|
218
|
-
raise ValueError(err)
|
211
|
+
@overload
|
212
|
+
async def aask_validate[T](
|
213
|
+
self,
|
214
|
+
question: List[str],
|
215
|
+
validator: Callable[[str], T | None],
|
216
|
+
default: None=None,
|
217
|
+
max_validations: PositiveInt = 2,
|
218
|
+
**kwargs: Unpack[GenerateKwargs],
|
219
|
+
) -> List[Optional[T]]: ...
|
219
220
|
|
220
|
-
async def
|
221
|
+
async def aask_validate[T](
|
221
222
|
self,
|
222
|
-
|
223
|
+
question: str | List[str],
|
223
224
|
validator: Callable[[str], T | None],
|
225
|
+
default: Optional[T] = None,
|
226
|
+
max_validations: PositiveInt = 2,
|
224
227
|
**kwargs: Unpack[GenerateKwargs],
|
225
|
-
) -> List[T]:
|
226
|
-
"""Asynchronously asks a
|
228
|
+
) -> Optional[T] | List[Optional[T]] | List[T] | T:
|
229
|
+
"""Asynchronously asks a question and validates the response using a given validator.
|
227
230
|
|
228
231
|
Args:
|
229
|
-
|
232
|
+
question (str): The question to ask.
|
230
233
|
validator (Callable[[str], T | None]): A function to validate the response.
|
231
|
-
|
234
|
+
default (T | None): Default value to return if validation fails. Defaults to None.
|
235
|
+
max_validations (PositiveInt): Maximum number of validation attempts. Defaults to 2.
|
236
|
+
**kwargs (Unpack[LLMKwargs]): Additional keyword arguments for the LLM usage.
|
232
237
|
|
233
238
|
Returns:
|
234
239
|
T: The validated response.
|
235
240
|
|
236
|
-
Raises:
|
237
|
-
ValueError: If the response fails to validate after the maximum number of attempts.
|
238
241
|
"""
|
239
|
-
return await gather(*[self.aask_validate(question, validator, **kwargs) for question in questions])
|
240
242
|
|
241
|
-
|
243
|
+
async def _inner(q: str) -> Optional[T]:
|
244
|
+
for lap in range(max_validations):
|
245
|
+
try:
|
246
|
+
if (response := await self.aask(question=q, **kwargs)) and (validated := validator(response)):
|
247
|
+
logger.debug(f"Successfully validated the response at {lap}th attempt.")
|
248
|
+
return validated
|
249
|
+
except Exception as e: # noqa: BLE001
|
250
|
+
logger.error(f"Error during validation: \n{e}")
|
251
|
+
break
|
252
|
+
kwargs["no_cache"] = True
|
253
|
+
logger.debug("Closed the cache for the next attempt")
|
254
|
+
if default is None:
|
255
|
+
logger.error(f"Failed to validate the response after {max_validations} attempts.")
|
256
|
+
return default
|
257
|
+
|
258
|
+
if isinstance(question, str):
|
259
|
+
return await _inner(question)
|
260
|
+
|
261
|
+
return await gather(*[_inner(q) for q in question])
|
262
|
+
|
263
|
+
async def aliststr(
|
264
|
+
self, requirement: str, k: NonNegativeInt = 0, **kwargs: Unpack[ValidateKwargs[List[str]]]
|
265
|
+
) -> List[str]:
|
242
266
|
"""Asynchronously generates a list of strings based on a given requirement.
|
243
267
|
|
244
268
|
Args:
|
245
269
|
requirement (str): The requirement for the list of strings.
|
246
270
|
k (NonNegativeInt): The number of choices to select, 0 means infinite. Defaults to 0.
|
247
|
-
**kwargs (Unpack[
|
271
|
+
**kwargs (Unpack[ValidateKwargs]): Additional keyword arguments for the LLM usage.
|
248
272
|
|
249
273
|
Returns:
|
250
274
|
List[str]: The validated response as a list of strings.
|
@@ -258,12 +282,48 @@ class LLMUsage(ScopedConfig):
|
|
258
282
|
**kwargs,
|
259
283
|
)
|
260
284
|
|
285
|
+
async def apathstr(self, requirement: str, **kwargs: Unpack[ChooseKwargs[List[str]]]) -> List[str]:
|
286
|
+
"""Asynchronously generates a list of strings based on a given requirement.
|
287
|
+
|
288
|
+
Args:
|
289
|
+
requirement (str): The requirement for the list of strings.
|
290
|
+
**kwargs (Unpack[ChooseKwargs]): Additional keyword arguments for the LLM usage.
|
291
|
+
|
292
|
+
Returns:
|
293
|
+
List[str]: The validated response as a list of strings.
|
294
|
+
"""
|
295
|
+
return await self.aliststr(
|
296
|
+
template_manager.render_template(
|
297
|
+
configs.templates.pathstr_template,
|
298
|
+
{"requirement": requirement},
|
299
|
+
),
|
300
|
+
**kwargs,
|
301
|
+
)
|
302
|
+
|
303
|
+
async def awhich_pathstr(self, requirement: str, **kwargs: Unpack[ValidateKwargs[List[str]]]) -> str:
|
304
|
+
"""Asynchronously generates a single path string based on a given requirement.
|
305
|
+
|
306
|
+
Args:
|
307
|
+
requirement (str): The requirement for the list of strings.
|
308
|
+
**kwargs (Unpack[ValidateKwargs]): Additional keyword arguments for the LLM usage.
|
309
|
+
|
310
|
+
Returns:
|
311
|
+
str: The validated response as a single string.
|
312
|
+
"""
|
313
|
+
return (
|
314
|
+
await self.apathstr(
|
315
|
+
requirement,
|
316
|
+
k=1,
|
317
|
+
**kwargs,
|
318
|
+
)
|
319
|
+
).pop()
|
320
|
+
|
261
321
|
async def achoose[T: WithBriefing](
|
262
322
|
self,
|
263
323
|
instruction: str,
|
264
324
|
choices: List[T],
|
265
325
|
k: NonNegativeInt = 0,
|
266
|
-
**kwargs: Unpack[
|
326
|
+
**kwargs: Unpack[ValidateKwargs[List[T]]],
|
267
327
|
) -> List[T]:
|
268
328
|
"""Asynchronously executes a multi-choice decision-making process, generating a prompt based on the instruction and options, and validates the returned selection results.
|
269
329
|
|
@@ -271,7 +331,7 @@ class LLMUsage(ScopedConfig):
|
|
271
331
|
instruction (str): The user-provided instruction/question description.
|
272
332
|
choices (List[T]): A list of candidate options, requiring elements to have `name` and `briefing` fields.
|
273
333
|
k (NonNegativeInt): The number of choices to select, 0 means infinite. Defaults to 0.
|
274
|
-
**kwargs (Unpack[
|
334
|
+
**kwargs (Unpack[ValidateKwargs]): Additional keyword arguments for the LLM usage.
|
275
335
|
|
276
336
|
Returns:
|
277
337
|
List[T]: The final validated selection result list, with element types matching the input `choices`.
|
@@ -314,14 +374,14 @@ class LLMUsage(ScopedConfig):
|
|
314
374
|
self,
|
315
375
|
instruction: str,
|
316
376
|
choices: List[T],
|
317
|
-
**kwargs: Unpack[
|
377
|
+
**kwargs: Unpack[ValidateKwargs[List[T]]],
|
318
378
|
) -> T:
|
319
379
|
"""Asynchronously picks a single choice from a list of options using AI validation.
|
320
380
|
|
321
381
|
Args:
|
322
382
|
instruction (str): The user-provided instruction/question description.
|
323
383
|
choices (List[T]): A list of candidate options, requiring elements to have `name` and `briefing` fields.
|
324
|
-
**kwargs (Unpack[
|
384
|
+
**kwargs (Unpack[ValidateKwargs]): Additional keyword arguments for the LLM usage.
|
325
385
|
|
326
386
|
Returns:
|
327
387
|
T: The single selected item from the choices list.
|
@@ -343,7 +403,7 @@ class LLMUsage(ScopedConfig):
|
|
343
403
|
prompt: str,
|
344
404
|
affirm_case: str = "",
|
345
405
|
deny_case: str = "",
|
346
|
-
**kwargs: Unpack[
|
406
|
+
**kwargs: Unpack[ValidateKwargs[bool]],
|
347
407
|
) -> bool:
|
348
408
|
"""Asynchronously judges a prompt using AI validation.
|
349
409
|
|
@@ -351,7 +411,7 @@ class LLMUsage(ScopedConfig):
|
|
351
411
|
prompt (str): The input prompt to be judged.
|
352
412
|
affirm_case (str): The affirmative case for the AI model. Defaults to an empty string.
|
353
413
|
deny_case (str): The negative case for the AI model. Defaults to an empty string.
|
354
|
-
**kwargs (Unpack[
|
414
|
+
**kwargs (Unpack[ValidateKwargs]): Additional keyword arguments for the LLM usage.
|
355
415
|
|
356
416
|
Returns:
|
357
417
|
bool: The judgment result (True or False) based on the AI's response.
|
@@ -457,18 +517,13 @@ class ToolBoxUsage(LLMUsage):
|
|
457
517
|
async def choose_toolboxes(
|
458
518
|
self,
|
459
519
|
task: Task,
|
460
|
-
|
461
|
-
k: NonNegativeInt = 0,
|
462
|
-
max_validations: PositiveInt = 2,
|
463
|
-
**kwargs: Unpack[LLMKwargs],
|
520
|
+
**kwargs: Unpack[ChooseKwargs[List[ToolBox]]],
|
464
521
|
) -> List[ToolBox]:
|
465
522
|
"""Asynchronously executes a multi-choice decision-making process to choose toolboxes.
|
466
523
|
|
467
524
|
Args:
|
468
525
|
task (Task): The task for which to choose toolboxes.
|
469
526
|
system_message (str): Custom system-level prompt, defaults to an empty string.
|
470
|
-
k (NonNegativeInt): The number of toolboxes to select, 0 means infinite. Defaults to 0.
|
471
|
-
max_validations (PositiveInt): Maximum number of validation failures, default is 2.
|
472
527
|
**kwargs (Unpack[LLMKwargs]): Additional keyword arguments for the LLM usage.
|
473
528
|
|
474
529
|
Returns:
|
@@ -478,11 +533,8 @@ class ToolBoxUsage(LLMUsage):
|
|
478
533
|
logger.warning("No toolboxes available.")
|
479
534
|
return []
|
480
535
|
return await self.achoose(
|
481
|
-
instruction=task.briefing,
|
536
|
+
instruction=task.briefing,
|
482
537
|
choices=list(self.toolboxes),
|
483
|
-
k=k,
|
484
|
-
max_validations=max_validations,
|
485
|
-
system_message=system_message,
|
486
538
|
**kwargs,
|
487
539
|
)
|
488
540
|
|
@@ -490,19 +542,13 @@ class ToolBoxUsage(LLMUsage):
|
|
490
542
|
self,
|
491
543
|
task: Task,
|
492
544
|
toolbox: ToolBox,
|
493
|
-
|
494
|
-
k: NonNegativeInt = 0,
|
495
|
-
max_validations: PositiveInt = 2,
|
496
|
-
**kwargs: Unpack[LLMKwargs],
|
545
|
+
**kwargs: Unpack[ChooseKwargs[List[Tool]]],
|
497
546
|
) -> List[Tool]:
|
498
547
|
"""Asynchronously executes a multi-choice decision-making process to choose tools.
|
499
548
|
|
500
549
|
Args:
|
501
550
|
task (Task): The task for which to choose tools.
|
502
551
|
toolbox (ToolBox): The toolbox from which to choose tools.
|
503
|
-
system_message (str): Custom system-level prompt, defaults to an empty string.
|
504
|
-
k (NonNegativeInt): The number of tools to select, 0 means infinite. Defaults to 0.
|
505
|
-
max_validations (PositiveInt): Maximum number of validation failures, default is 2.
|
506
552
|
**kwargs (Unpack[LLMKwargs]): Additional keyword arguments for the LLM usage.
|
507
553
|
|
508
554
|
Returns:
|
@@ -512,11 +558,8 @@ class ToolBoxUsage(LLMUsage):
|
|
512
558
|
logger.warning(f"No tools available in toolbox {toolbox.name}.")
|
513
559
|
return []
|
514
560
|
return await self.achoose(
|
515
|
-
instruction=task.briefing,
|
561
|
+
instruction=task.briefing,
|
516
562
|
choices=toolbox.tools,
|
517
|
-
k=k,
|
518
|
-
max_validations=max_validations,
|
519
|
-
system_message=system_message,
|
520
563
|
**kwargs,
|
521
564
|
)
|
522
565
|
|
fabricatio/parser.py
CHANGED
@@ -1,12 +1,14 @@
|
|
1
1
|
"""A module to parse text using regular expressions."""
|
2
2
|
|
3
|
-
from typing import Any, Callable, Optional, Self, Tuple, Type
|
3
|
+
from typing import Any, Callable, Iterable, List, Optional, Self, Tuple, Type
|
4
4
|
|
5
5
|
import orjson
|
6
6
|
import regex
|
7
|
+
from json_repair import repair_json
|
7
8
|
from pydantic import BaseModel, ConfigDict, Field, PositiveInt, PrivateAttr, ValidationError
|
8
9
|
from regex import Pattern, compile
|
9
10
|
|
11
|
+
from fabricatio.config import configs
|
10
12
|
from fabricatio.journal import logger
|
11
13
|
|
12
14
|
|
@@ -25,12 +27,31 @@ class Capture(BaseModel):
|
|
25
27
|
"""The regular expression pattern to search for."""
|
26
28
|
flags: PositiveInt = Field(default=regex.DOTALL | regex.MULTILINE | regex.IGNORECASE, frozen=True)
|
27
29
|
"""The flags to use when compiling the regular expression pattern."""
|
30
|
+
capture_type: Optional[str] = None
|
31
|
+
"""The type of capture to perform, e.g., 'json', which is used to dispatch the fixer accordingly."""
|
28
32
|
_compiled: Pattern = PrivateAttr()
|
29
33
|
|
30
34
|
def model_post_init(self, __context: Any) -> None:
|
31
35
|
"""Initialize the compiled pattern."""
|
32
36
|
self._compiled = compile(self.pattern, self.flags)
|
33
37
|
|
38
|
+
def fix[T](self, text: str | Iterable[str]|T) -> str | List[str]|T:
|
39
|
+
"""Fix the text using the pattern.
|
40
|
+
|
41
|
+
Args:
|
42
|
+
text (str | List[str]): The text to fix.
|
43
|
+
|
44
|
+
Returns:
|
45
|
+
str | List[str]: The fixed text with the same type as input.
|
46
|
+
"""
|
47
|
+
match self.capture_type:
|
48
|
+
case "json":
|
49
|
+
if isinstance(text, str):
|
50
|
+
return repair_json(text,ensure_ascii=False)
|
51
|
+
return [repair_json(item) for item in text]
|
52
|
+
case _:
|
53
|
+
return text
|
54
|
+
|
34
55
|
def capture(self, text: str) -> Tuple[str, ...] | str | None:
|
35
56
|
"""Capture the first occurrence of the pattern in the given text.
|
36
57
|
|
@@ -44,12 +65,12 @@ class Capture(BaseModel):
|
|
44
65
|
match = self._compiled.search(text)
|
45
66
|
if match is None:
|
46
67
|
return None
|
47
|
-
|
68
|
+
groups = self.fix(match.groups()) if configs.general.use_json_repair else match.groups()
|
48
69
|
if self.target_groups:
|
49
|
-
cap = tuple(
|
70
|
+
cap = tuple(groups[g - 1] for g in self.target_groups)
|
50
71
|
logger.debug(f"Captured text: {'\n\n'.join(cap)}")
|
51
72
|
return cap
|
52
|
-
cap =
|
73
|
+
cap = groups[0]
|
53
74
|
logger.debug(f"Captured text: \n{cap}")
|
54
75
|
return cap
|
55
76
|
|
@@ -111,7 +132,7 @@ class Capture(BaseModel):
|
|
111
132
|
Returns:
|
112
133
|
Self: The instance of the class with the captured code block.
|
113
134
|
"""
|
114
|
-
return cls(pattern=f"```{language}\n(.*?)\n```")
|
135
|
+
return cls(pattern=f"```{language}\n(.*?)\n```", capture_type=language)
|
115
136
|
|
116
137
|
|
117
138
|
JsonCapture = Capture.capture_code_block("json")
|
fabricatio/toolboxes/__init__.py
CHANGED
@@ -5,13 +5,11 @@ from typing import Set
|
|
5
5
|
from fabricatio.models.tool import ToolBox
|
6
6
|
from fabricatio.toolboxes.arithmetic import arithmetic_toolbox
|
7
7
|
from fabricatio.toolboxes.fs import fs_toolbox
|
8
|
-
from fabricatio.toolboxes.task import task_toolbox
|
9
8
|
|
10
|
-
basic_toolboxes: Set[ToolBox] = {
|
9
|
+
basic_toolboxes: Set[ToolBox] = {arithmetic_toolbox}
|
11
10
|
|
12
11
|
__all__ = [
|
13
12
|
"arithmetic_toolbox",
|
14
13
|
"basic_toolboxes",
|
15
14
|
"fs_toolbox",
|
16
|
-
"task_toolbox",
|
17
15
|
]
|
fabricatio/toolboxes/fs.py
CHANGED
@@ -1,6 +1,18 @@
|
|
1
1
|
"""File system tool box."""
|
2
2
|
|
3
|
-
from fabricatio.fs
|
3
|
+
from fabricatio.fs import (
|
4
|
+
absolute_path,
|
5
|
+
copy_file,
|
6
|
+
create_directory,
|
7
|
+
delete_directory,
|
8
|
+
delete_file,
|
9
|
+
dump_text,
|
10
|
+
gather_files,
|
11
|
+
move_file,
|
12
|
+
safe_json_read,
|
13
|
+
safe_text_read,
|
14
|
+
tree,
|
15
|
+
)
|
4
16
|
from fabricatio.models.tool import ToolBox
|
5
17
|
|
6
18
|
fs_toolbox = (
|
@@ -12,4 +24,8 @@ fs_toolbox = (
|
|
12
24
|
.add_tool(tree)
|
13
25
|
.add_tool(delete_directory)
|
14
26
|
.add_tool(create_directory)
|
27
|
+
.add_tool(absolute_path)
|
28
|
+
.add_tool(safe_text_read)
|
29
|
+
.add_tool(safe_json_read)
|
30
|
+
.add_tool(gather_files)
|
15
31
|
)
|
fabricatio/workflows/articles.py
CHANGED
@@ -1,11 +1,15 @@
|
|
1
1
|
"""Store article essence in the database."""
|
2
2
|
|
3
|
-
from fabricatio.actions.article import
|
4
|
-
from fabricatio.actions.
|
3
|
+
from fabricatio.actions.article import GenerateArticleProposal, GenerateOutline
|
4
|
+
from fabricatio.actions.output import DumpFinalizedOutput
|
5
5
|
from fabricatio.models.action import WorkFlow
|
6
6
|
|
7
|
-
|
8
|
-
name="
|
9
|
-
description="
|
10
|
-
steps=(
|
7
|
+
WriteOutlineWorkFlow = WorkFlow(
|
8
|
+
name="Generate Article Outline",
|
9
|
+
description="Generate an outline for an article. dump the outline to the given path. in typst format.",
|
10
|
+
steps=(
|
11
|
+
GenerateArticleProposal,
|
12
|
+
GenerateOutline(output_key="to_dump"),
|
13
|
+
DumpFinalizedOutput(output_key="task_output"),
|
14
|
+
),
|
11
15
|
)
|
@@ -0,0 +1,11 @@
|
|
1
|
+
"""The workflow for extracting the essence of an article and storing it in the database."""
|
2
|
+
|
3
|
+
from fabricatio.actions.article import ExtractArticleEssence
|
4
|
+
from fabricatio.actions.rag import InjectToDB
|
5
|
+
from fabricatio.models.action import WorkFlow
|
6
|
+
|
7
|
+
StoreArticle = WorkFlow(
|
8
|
+
name="Extract Article Essence",
|
9
|
+
description="Extract the essence of an article in the given path, and store it in the database.",
|
10
|
+
steps=(ExtractArticleEssence(output_key="to_inject"), InjectToDB(output_key="task_output")),
|
11
|
+
)
|
Binary file
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: fabricatio
|
3
|
-
Version: 0.2.
|
3
|
+
Version: 0.2.5
|
4
4
|
Classifier: License :: OSI Approved :: MIT License
|
5
5
|
Classifier: Programming Language :: Rust
|
6
6
|
Classifier: Programming Language :: Python :: 3.12
|
@@ -11,6 +11,7 @@ Classifier: Typing :: Typed
|
|
11
11
|
Requires-Dist: appdirs>=1.4.4
|
12
12
|
Requires-Dist: asyncio>=3.4.3
|
13
13
|
Requires-Dist: asyncstdlib>=3.13.0
|
14
|
+
Requires-Dist: json-repair>=0.39.1
|
14
15
|
Requires-Dist: litellm>=1.60.0
|
15
16
|
Requires-Dist: loguru>=0.7.3
|
16
17
|
Requires-Dist: magika>=0.5.1
|
@@ -0,0 +1,41 @@
|
|
1
|
+
fabricatio-0.2.5.dist-info/METADATA,sha256=AOoGHQbgVdQW0GXyg5pXbONeKgMRLgSc-9XER1YWix4,8891
|
2
|
+
fabricatio-0.2.5.dist-info/WHEEL,sha256=tpW5AN9B-9qsM9WW2FXG2r193YXiqexDadpKp0A2daI,96
|
3
|
+
fabricatio-0.2.5.dist-info/licenses/LICENSE,sha256=do7J7EiCGbq0QPbMAL_FqLYufXpHnCnXBOuqVPwSV8Y,1088
|
4
|
+
fabricatio/actions/article.py,sha256=yzRwgc203vI3MW_oWyFybDxTz6kaBBvUgN2zOJJ9Amc,2825
|
5
|
+
fabricatio/actions/output.py,sha256=RWXulsfN_qrCFik0B6lGwXf6MMtgK3CaF1ENbK0l85o,684
|
6
|
+
fabricatio/actions/rag.py,sha256=wBygtCHVBIvRAhsQRz6pJv2-QDsGtV6fEo8IdOvfVoM,831
|
7
|
+
fabricatio/capabilities/propose.py,sha256=y3kge5g6bb8HYuV8e9h4MdqOMTlsfAIZpqE_cagWPTY,1593
|
8
|
+
fabricatio/capabilities/rag.py,sha256=YYaabej-f6a7gBRbJRcJ_Fj89BloNCw0sHOPqs_W8Bk,15773
|
9
|
+
fabricatio/capabilities/rating.py,sha256=0Xrfp_AW9u2ltHnZXxz3Pt3fr8fMsO0QiFadegh49IU,14406
|
10
|
+
fabricatio/capabilities/review.py,sha256=4FbM8dO-JSPnu22I1lykM9y1gzBtXI5Wj33KtzvULWU,9757
|
11
|
+
fabricatio/capabilities/task.py,sha256=qH3Zmq3FxM5bcnIGqTm55KyU5Mt001Ly7fqcaEHPHiU,4595
|
12
|
+
fabricatio/config.py,sha256=pZruV1dL-U1_ynb6_GoZuaKEnDqPmDMJ45ycWXEjlCw,14894
|
13
|
+
fabricatio/core.py,sha256=VQ_JKgUGIy2gZ8xsTBZCdr_IP7wC5aPg0_bsOmjQ588,6458
|
14
|
+
fabricatio/decorators.py,sha256=uzsP4tFKQNjDHBkofsjjoJA0IUAaYOtt6YVedoyOqlo,6551
|
15
|
+
fabricatio/fs/curd.py,sha256=N6l2MncjrFfnXBRtteRouXp5Rjy8EAKC_i29_G-zz98,4618
|
16
|
+
fabricatio/fs/readers.py,sha256=RNWx4T2XXigeprJOxw8YjHfyGlrCtB5MVz01Gw9lvw4,1202
|
17
|
+
fabricatio/fs/__init__.py,sha256=uk4yIlz43Yl2mQZ5QuoMstg0DaIQV9h6XXxU4SbdWpY,588
|
18
|
+
fabricatio/journal.py,sha256=pCpIFk-IOsDmCcjx1fOsWDDxjkeAGYgu2OrFnuItwcA,801
|
19
|
+
fabricatio/models/action.py,sha256=MZZ3jQcMcjKXJ8h5L-gedeJGPYghyTcRo_wF38g8LIE,6461
|
20
|
+
fabricatio/models/events.py,sha256=QvlnS8FEELg6KNabcJMeh2GV_y0ZBzKOPphcteKYWYU,4183
|
21
|
+
fabricatio/models/extra.py,sha256=O8ncZVsaNmlR5f8c_b2HJc-yVZQ2YhB6ddDbfT0Ysh4,7412
|
22
|
+
fabricatio/models/generic.py,sha256=diqQ3qwh6rlE7PjFTBYhVVCc3srYU-8R8D9Rk8ESOfM,13594
|
23
|
+
fabricatio/models/kwargs_types.py,sha256=RKD_EXMlz4vuVpGzB9y5qDotSHNOKjk6R0EI4BoNaCg,4282
|
24
|
+
fabricatio/models/role.py,sha256=M9JVDmLLvJhY3G0hHMdcs1ywJ0eoAb6CLYbzhv-l_8s,1839
|
25
|
+
fabricatio/models/task.py,sha256=sbC0EAZC4rPL2GCJ9i9GlFcZUJ96g39SQ2QP8bbgdqs,10492
|
26
|
+
fabricatio/models/tool.py,sha256=4b-v4WIC_LuLOKzzXL9bvKXr8vmGZ8O2uAFv5-1KRA0,7052
|
27
|
+
fabricatio/models/usages.py,sha256=badtIf2622DJGfrLnEMxDKaIAQSI0n2Kx1m7TeVFsGE,26681
|
28
|
+
fabricatio/models/utils.py,sha256=QI3bYrKBbzLbKvyzVrZXGcWq3trOOTE-hQAC_WNvjMg,4457
|
29
|
+
fabricatio/parser.py,sha256=pwYAoLbWYXtABcnV3s5GdBM0Ngczq-r7n0jKSnpdAxE,5838
|
30
|
+
fabricatio/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
31
|
+
fabricatio/toolboxes/arithmetic.py,sha256=WLqhY-Pikv11Y_0SGajwZx3WhsLNpHKf9drzAqOf_nY,1369
|
32
|
+
fabricatio/toolboxes/fs.py,sha256=l4L1CVxJmjw9Ld2XUpIlWfV0_Fu_2Og6d3E13I-S4aE,736
|
33
|
+
fabricatio/toolboxes/__init__.py,sha256=KBJi5OG_pExscdlM7Bnt_UF43j4I3Lv6G71kPVu4KQU,395
|
34
|
+
fabricatio/workflows/articles.py,sha256=RebdC_BzSXC-xsck5I9ccC_XIgfhtoeM8FZuhtVDn3U,580
|
35
|
+
fabricatio/workflows/rag.py,sha256=-YYp2tlE9Vtfgpg6ROpu6QVO8j8yVSPa6yDzlN3qVxs,520
|
36
|
+
fabricatio/_rust.pyi,sha256=pI747rOciunGuQZDvfC3O0A6pLyOiaHSa3A5kHuQO0E,3169
|
37
|
+
fabricatio/_rust_instances.py,sha256=RybIzB2kCoMeU_1kMm2JTz1ka8i524hAra66joDf--s,314
|
38
|
+
fabricatio/__init__.py,sha256=22KLRAVoJZUrK7gsDjU1NfiLCnZtV-qgx6TO2YMbuH8,1930
|
39
|
+
fabricatio/_rust.cp312-win_amd64.pyd,sha256=6oc15oD6KF9Q-1yeYuUKKN8kv4haUb5JXTkoGY0_QPU,1816064
|
40
|
+
fabricatio-0.2.5.data/scripts/tdown.exe,sha256=ZSK6gpbQmPFCkhdCFM7oGebLCOrqptLjzmoQ2EVXEhs,3395072
|
41
|
+
fabricatio-0.2.5.dist-info/RECORD,,
|
fabricatio/toolboxes/task.py
DELETED
@@ -1,6 +0,0 @@
|
|
1
|
-
"""This module contains the toolbox for tasks management."""
|
2
|
-
|
3
|
-
from fabricatio.models.task import Task
|
4
|
-
from fabricatio.models.tool import ToolBox
|
5
|
-
|
6
|
-
task_toolbox = ToolBox(name="TaskToolBox", description="A toolbox for tasks management.").add_tool(Task.simple_task)
|
Binary file
|