fabricatio 0.3.15.dev5__cp312-cp312-win_amd64.whl → 0.4.4__cp312-cp312-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fabricatio/__init__.py +7 -8
- fabricatio/actions/__init__.py +69 -1
- fabricatio/capabilities/__init__.py +59 -1
- fabricatio/models/__init__.py +47 -0
- fabricatio/rust.cp312-win_amd64.pyd +0 -0
- fabricatio/toolboxes/__init__.py +2 -1
- fabricatio/toolboxes/arithmetic.py +1 -1
- fabricatio/toolboxes/fs.py +2 -2
- fabricatio/workflows/__init__.py +9 -0
- fabricatio-0.4.4.data/scripts/tdown.exe +0 -0
- {fabricatio-0.3.15.dev5.dist-info → fabricatio-0.4.4.dist-info}/METADATA +48 -25
- fabricatio-0.4.4.dist-info/RECORD +15 -0
- fabricatio/actions/article.py +0 -415
- fabricatio/actions/article_rag.py +0 -407
- fabricatio/actions/fs.py +0 -25
- fabricatio/actions/output.py +0 -247
- fabricatio/actions/rag.py +0 -96
- fabricatio/actions/rules.py +0 -83
- fabricatio/capabilities/advanced_judge.py +0 -20
- fabricatio/capabilities/advanced_rag.py +0 -61
- fabricatio/capabilities/censor.py +0 -105
- fabricatio/capabilities/check.py +0 -212
- fabricatio/capabilities/correct.py +0 -228
- fabricatio/capabilities/extract.py +0 -74
- fabricatio/capabilities/propose.py +0 -65
- fabricatio/capabilities/rag.py +0 -264
- fabricatio/capabilities/rating.py +0 -404
- fabricatio/capabilities/review.py +0 -114
- fabricatio/capabilities/task.py +0 -113
- fabricatio/decorators.py +0 -253
- fabricatio/emitter.py +0 -177
- fabricatio/fs/__init__.py +0 -35
- fabricatio/fs/curd.py +0 -153
- fabricatio/fs/readers.py +0 -61
- fabricatio/journal.py +0 -12
- fabricatio/models/action.py +0 -263
- fabricatio/models/adv_kwargs_types.py +0 -63
- fabricatio/models/extra/__init__.py +0 -1
- fabricatio/models/extra/advanced_judge.py +0 -32
- fabricatio/models/extra/aricle_rag.py +0 -286
- fabricatio/models/extra/article_base.py +0 -488
- fabricatio/models/extra/article_essence.py +0 -98
- fabricatio/models/extra/article_main.py +0 -285
- fabricatio/models/extra/article_outline.py +0 -45
- fabricatio/models/extra/article_proposal.py +0 -52
- fabricatio/models/extra/patches.py +0 -20
- fabricatio/models/extra/problem.py +0 -165
- fabricatio/models/extra/rag.py +0 -98
- fabricatio/models/extra/rule.py +0 -51
- fabricatio/models/generic.py +0 -904
- fabricatio/models/kwargs_types.py +0 -121
- fabricatio/models/role.py +0 -156
- fabricatio/models/task.py +0 -310
- fabricatio/models/tool.py +0 -328
- fabricatio/models/usages.py +0 -791
- fabricatio/parser.py +0 -114
- fabricatio/rust.pyi +0 -846
- fabricatio/utils.py +0 -156
- fabricatio/workflows/articles.py +0 -24
- fabricatio/workflows/rag.py +0 -11
- fabricatio-0.3.15.dev5.data/scripts/tdown.exe +0 -0
- fabricatio-0.3.15.dev5.data/scripts/ttm.exe +0 -0
- fabricatio-0.3.15.dev5.dist-info/RECORD +0 -63
- {fabricatio-0.3.15.dev5.dist-info → fabricatio-0.4.4.dist-info}/WHEEL +0 -0
- {fabricatio-0.3.15.dev5.dist-info → fabricatio-0.4.4.dist-info}/licenses/LICENSE +0 -0
fabricatio/models/generic.py
DELETED
@@ -1,904 +0,0 @@
|
|
1
|
-
"""This module defines generic classes for models in the Fabricatio library, providing a foundation for various model functionalities."""
|
2
|
-
|
3
|
-
from abc import ABC, abstractmethod
|
4
|
-
from datetime import datetime
|
5
|
-
from pathlib import Path
|
6
|
-
from typing import Any, Callable, Dict, Iterable, List, Mapping, Optional, Self, Sequence, Type, Union, final, overload
|
7
|
-
|
8
|
-
import ujson
|
9
|
-
from fabricatio.fs import dump_text
|
10
|
-
from fabricatio.fs.readers import safe_text_read
|
11
|
-
from fabricatio.journal import logger
|
12
|
-
from fabricatio.rust import CONFIG, TEMPLATE_MANAGER, blake3_hash, detect_language
|
13
|
-
from fabricatio.utils import ok
|
14
|
-
from pydantic import (
|
15
|
-
BaseModel,
|
16
|
-
ConfigDict,
|
17
|
-
Field,
|
18
|
-
NonNegativeFloat,
|
19
|
-
PositiveFloat,
|
20
|
-
PositiveInt,
|
21
|
-
PrivateAttr,
|
22
|
-
SecretStr,
|
23
|
-
)
|
24
|
-
from pydantic.json_schema import GenerateJsonSchema, JsonSchemaValue
|
25
|
-
|
26
|
-
|
27
|
-
class Base(BaseModel, ABC):
|
28
|
-
"""Base class for all models with Pydantic configuration.
|
29
|
-
|
30
|
-
This class sets up the basic Pydantic configuration for all models in the Fabricatio library.
|
31
|
-
The `model_config` uses `use_attribute_docstrings=True` to ensure field descriptions are
|
32
|
-
pulled from the attribute's docstring instead of the default Pydantic behavior.
|
33
|
-
"""
|
34
|
-
|
35
|
-
model_config = ConfigDict(use_attribute_docstrings=True)
|
36
|
-
|
37
|
-
|
38
|
-
class Display(Base, ABC):
|
39
|
-
"""Class that provides formatted JSON representation utilities.
|
40
|
-
|
41
|
-
Provides methods to generate both pretty-printed and compact JSON representations of the model.
|
42
|
-
Used for debugging and logging purposes.
|
43
|
-
"""
|
44
|
-
|
45
|
-
def display(self) -> str:
|
46
|
-
"""Generate pretty-printed JSON representation.
|
47
|
-
|
48
|
-
Returns:
|
49
|
-
str: JSON string with 1-level indentation for readability
|
50
|
-
"""
|
51
|
-
return self.model_dump_json(indent=1, by_alias=True)
|
52
|
-
|
53
|
-
def compact(self) -> str:
|
54
|
-
"""Generate compact JSON representation.
|
55
|
-
|
56
|
-
Returns:
|
57
|
-
str: Minified JSON string without whitespace
|
58
|
-
"""
|
59
|
-
return self.model_dump_json(by_alias=True)
|
60
|
-
|
61
|
-
@staticmethod
|
62
|
-
def seq_display(seq: Iterable["Display"], compact: bool = False) -> str:
|
63
|
-
"""Generate formatted display for sequence of Display objects.
|
64
|
-
|
65
|
-
Args:
|
66
|
-
seq (Iterable[Display]): Sequence of objects to display
|
67
|
-
compact (bool): Use compact format instead of pretty print
|
68
|
-
|
69
|
-
Returns:
|
70
|
-
str: Combined display output with boundary markers
|
71
|
-
"""
|
72
|
-
return (
|
73
|
-
"--- Start of Extra Info Sequence ---"
|
74
|
-
+ "\n".join(d.compact() if compact else d.display() for d in seq)
|
75
|
-
+ "--- End of Extra Info Sequence ---"
|
76
|
-
)
|
77
|
-
|
78
|
-
|
79
|
-
class Named(Base, ABC):
|
80
|
-
"""Class that includes a name attribute.
|
81
|
-
|
82
|
-
This class adds a name attribute to models, which is intended to be a unique identifier.
|
83
|
-
"""
|
84
|
-
|
85
|
-
name: str
|
86
|
-
"""The name of this object,briefly and conclusively."""
|
87
|
-
|
88
|
-
|
89
|
-
class Described(Base, ABC):
|
90
|
-
"""Class that includes a description attribute.
|
91
|
-
|
92
|
-
This class adds a description attribute to models, providing additional context or information.
|
93
|
-
"""
|
94
|
-
|
95
|
-
description: str
|
96
|
-
"""A comprehensive description of this object, including its purpose, scope, and context.
|
97
|
-
This should clearly explain what this object is about, why it exists, and in what situations
|
98
|
-
it applies. The description should be detailed enough to provide full understanding of
|
99
|
-
this object's intent and application."""
|
100
|
-
|
101
|
-
|
102
|
-
class Titled(Base, ABC):
|
103
|
-
"""Class that includes a title attribute."""
|
104
|
-
|
105
|
-
title: str
|
106
|
-
"""The title of this object, make it professional and concise.No prefixed heading number should be included."""
|
107
|
-
|
108
|
-
|
109
|
-
class WordCount(Base, ABC):
|
110
|
-
"""Class that includes a word count attribute."""
|
111
|
-
|
112
|
-
expected_word_count: int
|
113
|
-
"""Expected word count of this research component."""
|
114
|
-
|
115
|
-
@property
|
116
|
-
def exact_word_count(self) -> int:
|
117
|
-
"""Get the exact word count of this research component."""
|
118
|
-
raise NotImplementedError(f"`exact_word_count` is not implemented for {self.__class__.__name__}")
|
119
|
-
|
120
|
-
|
121
|
-
class FromMapping:
|
122
|
-
"""Class that provides a method to generate a list of objects from a mapping."""
|
123
|
-
|
124
|
-
@classmethod
|
125
|
-
@abstractmethod
|
126
|
-
def from_mapping[S](cls: S, mapping: Mapping[str, Any], **kwargs: Any) -> List[S]:
|
127
|
-
"""Generate a list of objects from a mapping."""
|
128
|
-
|
129
|
-
|
130
|
-
class FromSequence:
|
131
|
-
"""Class that provides a method to generate a list of objects from a sequence."""
|
132
|
-
|
133
|
-
@classmethod
|
134
|
-
@abstractmethod
|
135
|
-
def from_sequence[S](cls: S, sequence: Sequence[Any], **kwargs: Any) -> List[S]:
|
136
|
-
"""Generate a list of objects from a sequence."""
|
137
|
-
|
138
|
-
|
139
|
-
class AsPrompt:
|
140
|
-
"""Class that provides a method to generate a prompt from the model.
|
141
|
-
|
142
|
-
This class includes a method to generate a prompt based on the model's attributes.
|
143
|
-
"""
|
144
|
-
|
145
|
-
@final
|
146
|
-
def as_prompt(self) -> str:
|
147
|
-
"""Generate a prompt from the model.
|
148
|
-
|
149
|
-
Returns:
|
150
|
-
str: The generated prompt.
|
151
|
-
"""
|
152
|
-
return TEMPLATE_MANAGER.render_template(
|
153
|
-
CONFIG.templates.as_prompt_template,
|
154
|
-
self._as_prompt_inner(),
|
155
|
-
)
|
156
|
-
|
157
|
-
@abstractmethod
|
158
|
-
def _as_prompt_inner(self) -> Dict[str, str]:
|
159
|
-
"""Generate the inner part of the prompt.
|
160
|
-
|
161
|
-
This method should be implemented by subclasses to provide the specific data for the prompt.
|
162
|
-
|
163
|
-
Returns:
|
164
|
-
Dict[str, str]: The data for the prompt.
|
165
|
-
"""
|
166
|
-
|
167
|
-
|
168
|
-
class WithRef[T](Base, ABC):
|
169
|
-
"""Class that provides a reference to another object.
|
170
|
-
|
171
|
-
This class manages a reference to another object, allowing for easy access and updates.
|
172
|
-
"""
|
173
|
-
|
174
|
-
_reference: Optional[T] = PrivateAttr(None)
|
175
|
-
|
176
|
-
@property
|
177
|
-
def referenced(self) -> T:
|
178
|
-
"""Get the referenced object.
|
179
|
-
|
180
|
-
Returns:
|
181
|
-
T: The referenced object.
|
182
|
-
|
183
|
-
Raises:
|
184
|
-
ValueError: If the reference is not set.
|
185
|
-
"""
|
186
|
-
return ok(
|
187
|
-
self._reference, f"`{self.__class__.__name__}`'s `_reference` field is None. Have you called `update_ref`?"
|
188
|
-
)
|
189
|
-
|
190
|
-
@overload
|
191
|
-
def update_ref[S: WithRef](self: S, reference: T) -> S: ...
|
192
|
-
|
193
|
-
@overload
|
194
|
-
def update_ref[S: WithRef](self: S, reference: "WithRef[T]") -> S: ...
|
195
|
-
|
196
|
-
@overload
|
197
|
-
def update_ref[S: WithRef](self: S, reference: None = None) -> S: ...
|
198
|
-
|
199
|
-
def update_ref[S: WithRef](self: S, reference: Union[T, "WithRef[T]", None] = None) -> S:
|
200
|
-
"""Update the reference of the object.
|
201
|
-
|
202
|
-
Args:
|
203
|
-
reference (Union[T, WithRef[T], None]): The new reference to set.
|
204
|
-
|
205
|
-
Returns:
|
206
|
-
S: The current instance with the updated reference.
|
207
|
-
"""
|
208
|
-
if isinstance(reference, self.__class__):
|
209
|
-
self._reference = reference.referenced
|
210
|
-
else:
|
211
|
-
self._reference = reference # pyright: ignore [reportAttributeAccessIssue]
|
212
|
-
return self
|
213
|
-
|
214
|
-
|
215
|
-
class Language:
|
216
|
-
"""Class that provides a language attribute."""
|
217
|
-
|
218
|
-
@property
|
219
|
-
def language(self) -> str:
|
220
|
-
"""Get the language of the object."""
|
221
|
-
if isinstance(self, Described) and self.description:
|
222
|
-
return detect_language(self.description)
|
223
|
-
if isinstance(self, Titled) and self.title:
|
224
|
-
return detect_language(self.title)
|
225
|
-
if isinstance(self, Named) and self.name:
|
226
|
-
return detect_language(self.name)
|
227
|
-
raise RuntimeError(f"Cannot determine language! class that not support language: {self.__class__.__name__}")
|
228
|
-
|
229
|
-
|
230
|
-
class ModelHash(Base, ABC):
|
231
|
-
"""Class that provides a hash value for the object.
|
232
|
-
|
233
|
-
This class includes a method to calculate a hash value for the object based on its JSON representation.
|
234
|
-
"""
|
235
|
-
|
236
|
-
def __hash__(self) -> int:
|
237
|
-
"""Calculates a hash value for the object based on its model_dump_json representation.
|
238
|
-
|
239
|
-
Returns:
|
240
|
-
int: The hash value of the object.
|
241
|
-
"""
|
242
|
-
return hash(self.model_dump_json())
|
243
|
-
|
244
|
-
|
245
|
-
class UpdateFrom(ABC):
|
246
|
-
"""Class that provides a method to update the object from another object.
|
247
|
-
|
248
|
-
This class includes methods to update the current object with the attributes of another object.
|
249
|
-
"""
|
250
|
-
|
251
|
-
def update_pre_check(self, other: Self) -> Self:
|
252
|
-
"""Pre-check for updating the object from another object.
|
253
|
-
|
254
|
-
Args:
|
255
|
-
other (Self): The other object to update from.
|
256
|
-
|
257
|
-
Returns:
|
258
|
-
Self: The current instance after pre-check.
|
259
|
-
|
260
|
-
Raises:
|
261
|
-
TypeError: If the other object is not of the same type.
|
262
|
-
"""
|
263
|
-
if not isinstance(other, self.__class__):
|
264
|
-
raise TypeError(f"Cannot update from a non-{self.__class__.__name__} instance.")
|
265
|
-
|
266
|
-
return self
|
267
|
-
|
268
|
-
@abstractmethod
|
269
|
-
def update_from_inner(self, other: Self) -> Self:
|
270
|
-
"""Updates the current instance with the attributes of another instance.
|
271
|
-
|
272
|
-
This method should be implemented by subclasses to provide the specific update logic.
|
273
|
-
|
274
|
-
Args:
|
275
|
-
other (Self): The other instance to update from.
|
276
|
-
|
277
|
-
Returns:
|
278
|
-
Self: The current instance with updated attributes.
|
279
|
-
"""
|
280
|
-
|
281
|
-
@final
|
282
|
-
def update_from(self, other: Self) -> Self:
|
283
|
-
"""Updates the current instance with the attributes of another instance.
|
284
|
-
|
285
|
-
Args:
|
286
|
-
other (Self): The other instance to update from.
|
287
|
-
|
288
|
-
Returns:
|
289
|
-
Self: The current instance with updated attributes.
|
290
|
-
"""
|
291
|
-
return self.update_pre_check(other).update_from_inner(other)
|
292
|
-
|
293
|
-
|
294
|
-
class Introspect(ABC):
|
295
|
-
"""Class that provides a method to introspect the object.
|
296
|
-
|
297
|
-
This class includes a method to perform internal introspection of the object.
|
298
|
-
"""
|
299
|
-
|
300
|
-
@abstractmethod
|
301
|
-
def introspect(self) -> str:
|
302
|
-
"""Internal introspection of the object.
|
303
|
-
|
304
|
-
Returns:
|
305
|
-
str: The internal introspection of the object.
|
306
|
-
"""
|
307
|
-
|
308
|
-
|
309
|
-
class WithBriefing(Named, Described, ABC):
|
310
|
-
"""Class that provides a briefing based on the name and description.
|
311
|
-
|
312
|
-
This class combines the name and description attributes to provide a brief summary of the object.
|
313
|
-
"""
|
314
|
-
|
315
|
-
@property
|
316
|
-
def briefing(self) -> str:
|
317
|
-
"""Get the briefing of the object.
|
318
|
-
|
319
|
-
Returns:
|
320
|
-
str: The briefing of the object.
|
321
|
-
"""
|
322
|
-
return f"{self.name}: {self.description}" if self.description else self.name
|
323
|
-
|
324
|
-
|
325
|
-
class UnsortGenerate(GenerateJsonSchema):
|
326
|
-
"""Class that provides a reverse JSON schema of the model.
|
327
|
-
|
328
|
-
This class overrides the sorting behavior of the JSON schema generation to maintain the original order.
|
329
|
-
"""
|
330
|
-
|
331
|
-
def sort(self, value: JsonSchemaValue, parent_key: str | None = None) -> JsonSchemaValue:
|
332
|
-
"""Not sort.
|
333
|
-
|
334
|
-
Args:
|
335
|
-
value (JsonSchemaValue): The JSON schema value to sort.
|
336
|
-
parent_key (str | None): The parent key of the JSON schema value.
|
337
|
-
|
338
|
-
Returns:
|
339
|
-
JsonSchemaValue: The JSON schema value without sorting.
|
340
|
-
"""
|
341
|
-
return value
|
342
|
-
|
343
|
-
|
344
|
-
class WithFormatedJsonSchema(Base, ABC):
|
345
|
-
"""Class that provides a formatted JSON schema of the model.
|
346
|
-
|
347
|
-
This class includes a method to generate a formatted JSON schema of the model.
|
348
|
-
"""
|
349
|
-
|
350
|
-
@classmethod
|
351
|
-
def formated_json_schema(cls) -> str:
|
352
|
-
"""Get the JSON schema of the model in a formatted string.
|
353
|
-
|
354
|
-
Returns:
|
355
|
-
str: The JSON schema of the model in a formatted string.
|
356
|
-
"""
|
357
|
-
return ujson.dumps(
|
358
|
-
cls.model_json_schema(schema_generator=UnsortGenerate), indent=2, ensure_ascii=False, sort_keys=False
|
359
|
-
)
|
360
|
-
|
361
|
-
|
362
|
-
class CreateJsonObjPrompt(WithFormatedJsonSchema, ABC):
|
363
|
-
"""Class that provides a prompt for creating a JSON object.
|
364
|
-
|
365
|
-
This class includes a method to create a prompt for creating a JSON object based on the model's schema and a requirement.
|
366
|
-
"""
|
367
|
-
|
368
|
-
@classmethod
|
369
|
-
@overload
|
370
|
-
def create_json_prompt(cls, requirement: List[str]) -> List[str]: ...
|
371
|
-
|
372
|
-
@classmethod
|
373
|
-
@overload
|
374
|
-
def create_json_prompt(cls, requirement: str) -> str: ...
|
375
|
-
|
376
|
-
@classmethod
|
377
|
-
def create_json_prompt(cls, requirement: str | List[str]) -> str | List[str]:
|
378
|
-
"""Create the prompt for creating a JSON object with given requirement.
|
379
|
-
|
380
|
-
Args:
|
381
|
-
requirement (str | List[str]): The requirement for the JSON object.
|
382
|
-
|
383
|
-
Returns:
|
384
|
-
str | List[str]: The prompt for creating a JSON object with given requirement.
|
385
|
-
"""
|
386
|
-
if isinstance(requirement, str):
|
387
|
-
return TEMPLATE_MANAGER.render_template(
|
388
|
-
CONFIG.templates.create_json_obj_template,
|
389
|
-
{"requirement": requirement, "json_schema": cls.formated_json_schema()},
|
390
|
-
)
|
391
|
-
return [
|
392
|
-
TEMPLATE_MANAGER.render_template(
|
393
|
-
CONFIG.templates.create_json_obj_template,
|
394
|
-
{"requirement": r, "json_schema": cls.formated_json_schema()},
|
395
|
-
)
|
396
|
-
for r in requirement
|
397
|
-
]
|
398
|
-
|
399
|
-
|
400
|
-
class InstantiateFromString(Base, ABC):
|
401
|
-
"""Class that provides a method to instantiate the class from a string.
|
402
|
-
|
403
|
-
This class includes a method to instantiate the class from a JSON string representation.
|
404
|
-
"""
|
405
|
-
|
406
|
-
@classmethod
|
407
|
-
def instantiate_from_string(cls, string: str) -> Self | None:
|
408
|
-
"""Instantiate the class from a string.
|
409
|
-
|
410
|
-
Args:
|
411
|
-
string (str): The string to instantiate the class from.
|
412
|
-
|
413
|
-
Returns:
|
414
|
-
Self | None: The instance of the class or None if the string is not valid.
|
415
|
-
"""
|
416
|
-
from fabricatio.parser import JsonCapture
|
417
|
-
|
418
|
-
obj = JsonCapture.convert_with(string, cls.model_validate_json)
|
419
|
-
logger.debug(f"Instantiate `{cls.__name__}` from string, {'Failed' if obj is None else 'Success'}.")
|
420
|
-
return obj
|
421
|
-
|
422
|
-
|
423
|
-
class ProposedAble(CreateJsonObjPrompt, InstantiateFromString, ABC):
|
424
|
-
"""Class that provides a method to propose a JSON object based on the requirement.
|
425
|
-
|
426
|
-
This class combines the functionality to create a prompt for a JSON object and instantiate it from a string.
|
427
|
-
"""
|
428
|
-
|
429
|
-
|
430
|
-
class SketchedAble(ProposedAble, Display, ABC):
|
431
|
-
"""Class that provides a method to scratch the object.
|
432
|
-
|
433
|
-
This class combines the functionality to propose a JSON object, instantiate it from a string, and display it.
|
434
|
-
"""
|
435
|
-
|
436
|
-
|
437
|
-
class ProposedUpdateAble(SketchedAble, UpdateFrom, ABC):
|
438
|
-
"""Make the obj can be updated from the proposed obj in place.
|
439
|
-
|
440
|
-
This class provides the ability to update an object in place from a proposed object.
|
441
|
-
"""
|
442
|
-
|
443
|
-
|
444
|
-
class FinalizedDumpAble(Base, ABC):
|
445
|
-
"""Class that provides a method to finalize the dump of the object.
|
446
|
-
|
447
|
-
This class includes methods to finalize the JSON representation of the object and dump it to a file.
|
448
|
-
"""
|
449
|
-
|
450
|
-
def finalized_dump(self) -> str:
|
451
|
-
"""Finalize the dump of the object.
|
452
|
-
|
453
|
-
Returns:
|
454
|
-
str: The finalized dump of the object.
|
455
|
-
"""
|
456
|
-
return self.model_dump_json(indent=1, by_alias=True)
|
457
|
-
|
458
|
-
def finalized_dump_to(self, path: str | Path) -> Self:
|
459
|
-
"""Finalize the dump of the object to a file.
|
460
|
-
|
461
|
-
Args:
|
462
|
-
path (str | Path): The path to save the finalized dump.
|
463
|
-
|
464
|
-
Returns:
|
465
|
-
Self: The current instance of the object.
|
466
|
-
"""
|
467
|
-
dump_text(path, self.finalized_dump())
|
468
|
-
return self
|
469
|
-
|
470
|
-
|
471
|
-
class WithDependency(Base, ABC):
|
472
|
-
"""Class that manages file dependencies.
|
473
|
-
|
474
|
-
This class includes methods to manage file dependencies required for reading or writing.
|
475
|
-
"""
|
476
|
-
|
477
|
-
dependencies: List[str] = Field(default_factory=list)
|
478
|
-
"""The file dependencies which is needed to read or write to meet a specific requirement, a list of file paths."""
|
479
|
-
|
480
|
-
def add_dependency[P: str | Path](self, dependency: P | List[P]) -> Self:
|
481
|
-
"""Add a file dependency to the task.
|
482
|
-
|
483
|
-
Args:
|
484
|
-
dependency (str | Path | List[str | Path]): The file dependency to add to the task.
|
485
|
-
|
486
|
-
Returns:
|
487
|
-
Self: The current instance of the task.
|
488
|
-
"""
|
489
|
-
if not isinstance(dependency, list):
|
490
|
-
dependency = [dependency]
|
491
|
-
self.dependencies.extend(Path(d).as_posix() for d in dependency)
|
492
|
-
return self
|
493
|
-
|
494
|
-
def remove_dependency[P: str | Path](self, dependency: P | List[P]) -> Self:
|
495
|
-
"""Remove a file dependency from the task.
|
496
|
-
|
497
|
-
Args:
|
498
|
-
dependency (str | Path | List[str | Path]): The file dependency to remove from the task.
|
499
|
-
|
500
|
-
Returns:
|
501
|
-
Self: The current instance of the task.
|
502
|
-
"""
|
503
|
-
if not isinstance(dependency, list):
|
504
|
-
dependency = [dependency]
|
505
|
-
for d in dependency:
|
506
|
-
self.dependencies.remove(Path(d).as_posix())
|
507
|
-
return self
|
508
|
-
|
509
|
-
def clear_dependencies(self) -> Self:
|
510
|
-
"""Clear all file dependencies from the task.
|
511
|
-
|
512
|
-
Returns:
|
513
|
-
Self: The current instance of the task.
|
514
|
-
"""
|
515
|
-
self.dependencies.clear()
|
516
|
-
return self
|
517
|
-
|
518
|
-
def override_dependencies[P: str | Path](self, dependencies: List[P] | P) -> Self:
|
519
|
-
"""Override the file dependencies of the task.
|
520
|
-
|
521
|
-
Args:
|
522
|
-
dependencies (List[str | Path] | str | Path): The file dependencies to override the task's dependencies.
|
523
|
-
|
524
|
-
Returns:
|
525
|
-
Self: The current instance of the task.
|
526
|
-
"""
|
527
|
-
return self.clear_dependencies().add_dependency(dependencies)
|
528
|
-
|
529
|
-
def pop_dependence[T](self, idx: int = -1, reader: Callable[[str], T] = safe_text_read) -> T:
|
530
|
-
"""Pop the file dependencies from the task.
|
531
|
-
|
532
|
-
Returns:
|
533
|
-
str: The popped file dependency
|
534
|
-
"""
|
535
|
-
return reader(self.dependencies.pop(idx))
|
536
|
-
|
537
|
-
@property
|
538
|
-
def dependencies_prompt(self) -> str:
|
539
|
-
"""Generate a prompt for the task based on the file dependencies.
|
540
|
-
|
541
|
-
Returns:
|
542
|
-
str: The generated prompt for the task.
|
543
|
-
"""
|
544
|
-
from fabricatio.fs import MAGIKA
|
545
|
-
|
546
|
-
return TEMPLATE_MANAGER.render_template(
|
547
|
-
CONFIG.templates.dependencies_template,
|
548
|
-
{
|
549
|
-
(pth := Path(p)).name: {
|
550
|
-
"path": pth.as_posix(),
|
551
|
-
"exists": pth.exists(),
|
552
|
-
"description": (identity := MAGIKA.identify_path(pth)).output.description,
|
553
|
-
"size": f"{pth.stat().st_size / (1024 * 1024) if pth.exists() and pth.is_file() else 0:.3f} MB",
|
554
|
-
"content": (text := safe_text_read(pth)),
|
555
|
-
"lines": len(text.splitlines()),
|
556
|
-
"language": identity.output.ct_label,
|
557
|
-
"checksum": blake3_hash(pth.read_bytes()) if pth.exists() and pth.is_file() else "unknown",
|
558
|
-
}
|
559
|
-
for p in self.dependencies
|
560
|
-
},
|
561
|
-
)
|
562
|
-
|
563
|
-
|
564
|
-
class Vectorizable(ABC):
|
565
|
-
"""Class that prepares the vectorization of the model.
|
566
|
-
|
567
|
-
This class includes methods to prepare the model for vectorization, ensuring it fits within a specified token length.
|
568
|
-
"""
|
569
|
-
|
570
|
-
@abstractmethod
|
571
|
-
def _prepare_vectorization_inner(self) -> str:
|
572
|
-
"""Prepare the model for vectorization."""
|
573
|
-
|
574
|
-
@final
|
575
|
-
def prepare_vectorization(self, max_length: Optional[int] = None) -> str:
|
576
|
-
"""Prepare the vectorization of the model.
|
577
|
-
|
578
|
-
Args:
|
579
|
-
max_length (Optional[int]): The maximum token length for the vectorization. Defaults to the configuration.
|
580
|
-
|
581
|
-
Returns:
|
582
|
-
str: The prepared vectorization of the model.
|
583
|
-
|
584
|
-
Raises:
|
585
|
-
ValueError: If the chunk exceeds the maximum sequence length.
|
586
|
-
"""
|
587
|
-
from litellm.utils import token_counter
|
588
|
-
|
589
|
-
max_length = max_length or CONFIG.embedding.max_sequence_length
|
590
|
-
chunk = self._prepare_vectorization_inner()
|
591
|
-
if max_length and (length := token_counter(text=chunk)) > max_length:
|
592
|
-
raise ValueError(f"Chunk exceeds maximum sequence length {max_length}, got {length}, see \n{chunk}")
|
593
|
-
|
594
|
-
return chunk
|
595
|
-
|
596
|
-
|
597
|
-
class ScopedConfig(Base, ABC):
|
598
|
-
"""Configuration holder with hierarchical fallback mechanism.
|
599
|
-
|
600
|
-
Manages LLM, embedding, and vector database configurations with fallback logic.
|
601
|
-
Allows configuration values to be overridden in a hierarchical manner.
|
602
|
-
"""
|
603
|
-
|
604
|
-
llm_api_endpoint: Optional[str] = None
|
605
|
-
"""The OpenAI API endpoint."""
|
606
|
-
|
607
|
-
llm_api_key: Optional[SecretStr] = None
|
608
|
-
"""The OpenAI API key."""
|
609
|
-
|
610
|
-
llm_timeout: Optional[PositiveInt] = None
|
611
|
-
"""The timeout of the LLM model."""
|
612
|
-
|
613
|
-
llm_max_retries: Optional[PositiveInt] = None
|
614
|
-
"""The maximum number of retries."""
|
615
|
-
|
616
|
-
llm_model: Optional[str] = None
|
617
|
-
"""The LLM model name."""
|
618
|
-
|
619
|
-
llm_temperature: Optional[NonNegativeFloat] = None
|
620
|
-
"""The temperature of the LLM model."""
|
621
|
-
|
622
|
-
llm_stop_sign: Optional[str | List[str]] = None
|
623
|
-
"""The stop sign of the LLM model."""
|
624
|
-
|
625
|
-
llm_top_p: Optional[NonNegativeFloat] = None
|
626
|
-
"""The top p of the LLM model."""
|
627
|
-
|
628
|
-
llm_generation_count: Optional[PositiveInt] = None
|
629
|
-
"""The number of generations to generate."""
|
630
|
-
|
631
|
-
llm_stream: Optional[bool] = None
|
632
|
-
"""Whether to stream the LLM model's response."""
|
633
|
-
|
634
|
-
llm_max_tokens: Optional[PositiveInt] = None
|
635
|
-
"""The maximum number of tokens to generate."""
|
636
|
-
|
637
|
-
llm_tpm: Optional[PositiveInt] = None
|
638
|
-
"""The tokens per minute of the LLM model."""
|
639
|
-
|
640
|
-
llm_rpm: Optional[PositiveInt] = None
|
641
|
-
"""The requests per minute of the LLM model."""
|
642
|
-
|
643
|
-
llm_presence_penalty: Optional[PositiveFloat] = None
|
644
|
-
"""The presence penalty of the LLM model."""
|
645
|
-
|
646
|
-
llm_frequency_penalty: Optional[PositiveFloat] = None
|
647
|
-
"""The frequency penalty of the LLM model."""
|
648
|
-
|
649
|
-
embedding_api_endpoint: Optional[str] = None
|
650
|
-
"""The OpenAI API endpoint."""
|
651
|
-
|
652
|
-
embedding_api_key: Optional[SecretStr] = None
|
653
|
-
"""The OpenAI API key."""
|
654
|
-
|
655
|
-
embedding_timeout: Optional[PositiveInt] = None
|
656
|
-
"""The timeout of the LLM model."""
|
657
|
-
|
658
|
-
embedding_model: Optional[str] = None
|
659
|
-
"""The LLM model name."""
|
660
|
-
|
661
|
-
embedding_max_sequence_length: Optional[PositiveInt] = None
|
662
|
-
"""The maximum sequence length."""
|
663
|
-
|
664
|
-
embedding_dimensions: Optional[PositiveInt] = None
|
665
|
-
"""The dimensions of the embedding."""
|
666
|
-
|
667
|
-
embedding_caching: Optional[bool] = False
|
668
|
-
"""Whether to cache the embedding result."""
|
669
|
-
|
670
|
-
milvus_uri: Optional[str] = Field(default=None)
|
671
|
-
"""The URI of the Milvus server."""
|
672
|
-
|
673
|
-
milvus_token: Optional[SecretStr] = Field(default=None)
|
674
|
-
"""The token for the Milvus server."""
|
675
|
-
|
676
|
-
milvus_timeout: Optional[PositiveFloat] = Field(default=None)
|
677
|
-
"""The timeout for the Milvus server."""
|
678
|
-
|
679
|
-
milvus_dimensions: Optional[PositiveInt] = Field(default=None)
|
680
|
-
"""The dimensions of the Milvus server."""
|
681
|
-
|
682
|
-
@final
|
683
|
-
def fallback_to(self, other: Union["ScopedConfig", Any]) -> Self:
|
684
|
-
"""Merge configuration values with fallback priority.
|
685
|
-
|
686
|
-
Copies non-null values from 'other' to self where current values are None.
|
687
|
-
|
688
|
-
Args:
|
689
|
-
other (ScopedConfig): Configuration to fallback to
|
690
|
-
|
691
|
-
Returns:
|
692
|
-
Self: Current instance with merged values
|
693
|
-
"""
|
694
|
-
if not isinstance(other, ScopedConfig):
|
695
|
-
return self
|
696
|
-
|
697
|
-
# Iterate over the attribute names and copy values from 'other' to 'self' where applicable
|
698
|
-
# noinspection PydanticTypeChecker,PyTypeChecker
|
699
|
-
for attr_name in ScopedConfig.model_fields:
|
700
|
-
# Copy the attribute value from 'other' to 'self' only if 'self' has None and 'other' has a non-None value
|
701
|
-
if getattr(self, attr_name) is None and (attr := getattr(other, attr_name)) is not None:
|
702
|
-
setattr(self, attr_name, attr)
|
703
|
-
|
704
|
-
# Return the current instance to allow for method chaining
|
705
|
-
return self
|
706
|
-
|
707
|
-
@final
|
708
|
-
def hold_to(self, others: Union[Union["ScopedConfig", Any], Iterable[Union["ScopedConfig", Any]]]) -> Self:
|
709
|
-
"""Propagate non-null values to other configurations.
|
710
|
-
|
711
|
-
Copies current non-null values to target configurations where they are None.
|
712
|
-
|
713
|
-
Args:
|
714
|
-
others (ScopedConfig|Iterable): Target configurations to update
|
715
|
-
|
716
|
-
Returns:
|
717
|
-
Self: Current instance unchanged
|
718
|
-
"""
|
719
|
-
if not isinstance(others, Iterable):
|
720
|
-
others = [others]
|
721
|
-
|
722
|
-
for other in (o for o in others if isinstance(o, ScopedConfig)):
|
723
|
-
# noinspection PyTypeChecker,PydanticTypeChecker
|
724
|
-
for attr_name in ScopedConfig.model_fields:
|
725
|
-
if (attr := getattr(self, attr_name)) is not None and getattr(other, attr_name) is None:
|
726
|
-
setattr(other, attr_name, attr)
|
727
|
-
return self
|
728
|
-
|
729
|
-
|
730
|
-
class Patch[T](ProposedAble, ABC):
|
731
|
-
"""Base class for patches.
|
732
|
-
|
733
|
-
This class provides a base implementation for patches that can be applied to other objects.
|
734
|
-
"""
|
735
|
-
|
736
|
-
def apply(self, other: T) -> T:
|
737
|
-
"""Apply the patch to another instance.
|
738
|
-
|
739
|
-
Args:
|
740
|
-
other (T): The instance to apply the patch to.
|
741
|
-
|
742
|
-
Returns:
|
743
|
-
T: The instance with the patch applied.
|
744
|
-
|
745
|
-
Raises:
|
746
|
-
ValueError: If a field in the patch is not found in the target instance.
|
747
|
-
"""
|
748
|
-
for field in self.__class__.model_fields:
|
749
|
-
if not hasattr(other, field):
|
750
|
-
raise ValueError(f"{field} not found in {other}, are you applying to the wrong type?")
|
751
|
-
setattr(other, field, getattr(self, field))
|
752
|
-
return other
|
753
|
-
|
754
|
-
def as_kwargs(self) -> Dict[str, Any]:
|
755
|
-
"""Get the kwargs of the patch."""
|
756
|
-
return self.model_dump()
|
757
|
-
|
758
|
-
@staticmethod
|
759
|
-
def ref_cls() -> Optional[Type[BaseModel]]:
|
760
|
-
"""Get the reference class of the model."""
|
761
|
-
return None
|
762
|
-
|
763
|
-
@classmethod
|
764
|
-
def formated_json_schema(cls) -> str:
|
765
|
-
"""Get the JSON schema of the model in a formatted string.
|
766
|
-
|
767
|
-
Returns:
|
768
|
-
str: The JSON schema of the model in a formatted string.
|
769
|
-
"""
|
770
|
-
my_schema = cls.model_json_schema(schema_generator=UnsortGenerate)
|
771
|
-
|
772
|
-
ref_cls = cls.ref_cls()
|
773
|
-
if ref_cls is not None:
|
774
|
-
# copy the desc info of each corresponding fields from `ref_cls`
|
775
|
-
for field_name in [f for f in cls.model_fields if f in ref_cls.model_fields]:
|
776
|
-
my_schema["properties"][field_name]["description"] = (
|
777
|
-
ref_cls.model_fields[field_name].description or my_schema["properties"][field_name]["description"]
|
778
|
-
)
|
779
|
-
my_schema["description"] = ref_cls.__doc__
|
780
|
-
|
781
|
-
return ujson.dumps(my_schema, indent=2, ensure_ascii=False, sort_keys=False)
|
782
|
-
|
783
|
-
|
784
|
-
class SequencePatch[T](ProposedUpdateAble, ABC):
|
785
|
-
"""Base class for patches.
|
786
|
-
|
787
|
-
This class provides a base implementation for patches that can be applied to sequences of objects.
|
788
|
-
"""
|
789
|
-
|
790
|
-
tweaked: List[T]
|
791
|
-
"""Tweaked content list"""
|
792
|
-
|
793
|
-
def update_from_inner(self, other: Self) -> Self:
|
794
|
-
"""Updates the current instance with the attributes of another instance.
|
795
|
-
|
796
|
-
Args:
|
797
|
-
other (Self): The other instance to update from.
|
798
|
-
|
799
|
-
Returns:
|
800
|
-
Self: The current instance with updated attributes.
|
801
|
-
"""
|
802
|
-
self.tweaked.clear()
|
803
|
-
self.tweaked.extend(other.tweaked)
|
804
|
-
return self
|
805
|
-
|
806
|
-
@classmethod
|
807
|
-
def default(cls) -> Self:
|
808
|
-
"""Defaults to empty list.
|
809
|
-
|
810
|
-
Returns:
|
811
|
-
Self: A new instance with an empty list of tweaks.
|
812
|
-
"""
|
813
|
-
return cls(tweaked=[])
|
814
|
-
|
815
|
-
|
816
|
-
class PersistentAble(Base, ABC):
|
817
|
-
"""Class providing file persistence capabilities.
|
818
|
-
|
819
|
-
Enables saving model instances to disk with timestamped filenames and loading from persisted files.
|
820
|
-
Implements basic versioning through filename hashing and timestamping.
|
821
|
-
"""
|
822
|
-
|
823
|
-
def persist(self, path: str | Path) -> Self:
|
824
|
-
"""Save model instance to disk with versioned filename.
|
825
|
-
|
826
|
-
Args:
|
827
|
-
path (str | Path): Target directory or file path. If directory, filename is auto-generated.
|
828
|
-
|
829
|
-
Returns:
|
830
|
-
Self: Current instance for method chaining
|
831
|
-
|
832
|
-
Notes:
|
833
|
-
- Filename format: <ClassName>_<YYYYMMDD_HHMMSS>_<6-char_hash>.json
|
834
|
-
- Hash generated from JSON content ensures uniqueness
|
835
|
-
"""
|
836
|
-
p = Path(path)
|
837
|
-
out = self.model_dump_json(indent=1, by_alias=True)
|
838
|
-
|
839
|
-
# Generate a timestamp in the format YYYYMMDD_HHMMSS
|
840
|
-
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
841
|
-
|
842
|
-
# Generate the hash
|
843
|
-
file_hash = blake3_hash(out.encode())[:6]
|
844
|
-
|
845
|
-
# Construct the file name with timestamp and hash
|
846
|
-
file_name = f"{self.__class__.__name__}_{timestamp}_{file_hash}.json"
|
847
|
-
|
848
|
-
if p.is_dir():
|
849
|
-
p.joinpath(file_name).write_text(out, encoding="utf-8")
|
850
|
-
else:
|
851
|
-
p.mkdir(exist_ok=True, parents=True)
|
852
|
-
p.write_text(out, encoding="utf-8")
|
853
|
-
|
854
|
-
logger.info(f"Persisted `{self.__class__.__name__}` to {p.as_posix()}")
|
855
|
-
return self
|
856
|
-
|
857
|
-
@classmethod
|
858
|
-
def from_latest_persistent(cls, dir_path: str | Path) -> Optional[Self]:
|
859
|
-
"""Load most recent persisted instance from directory.
|
860
|
-
|
861
|
-
Args:
|
862
|
-
dir_path (str | Path): Directory containing persisted files
|
863
|
-
|
864
|
-
Returns:
|
865
|
-
Self: Most recently modified instance
|
866
|
-
|
867
|
-
Raises:
|
868
|
-
NotADirectoryError: If path is not a valid directory
|
869
|
-
FileNotFoundError: If no matching files found
|
870
|
-
"""
|
871
|
-
dir_path = Path(dir_path)
|
872
|
-
if not dir_path.is_dir():
|
873
|
-
return None
|
874
|
-
|
875
|
-
pattern = f"{cls.__name__}_*.json"
|
876
|
-
files = list(dir_path.glob(pattern))
|
877
|
-
|
878
|
-
if not files:
|
879
|
-
return None
|
880
|
-
|
881
|
-
def _get_timestamp(file_path: Path) -> datetime:
|
882
|
-
stem = file_path.stem
|
883
|
-
parts = stem.split("_")
|
884
|
-
return datetime.strptime(f"{parts[1]}_{parts[2]}", "%Y%m%d_%H%M%S")
|
885
|
-
|
886
|
-
files.sort(key=lambda f: _get_timestamp(f), reverse=True)
|
887
|
-
|
888
|
-
return cls.from_persistent(files.pop(0))
|
889
|
-
|
890
|
-
@classmethod
|
891
|
-
def from_persistent(cls, path: str | Path) -> Self:
|
892
|
-
"""Load an instance from a specific persisted file.
|
893
|
-
|
894
|
-
Args:
|
895
|
-
path (str | Path): Path to the JSON file.
|
896
|
-
|
897
|
-
Returns:
|
898
|
-
Self: The loaded instance from the file.
|
899
|
-
|
900
|
-
Raises:
|
901
|
-
FileNotFoundError: If the specified file does not exist.
|
902
|
-
ValueError: If the file content is invalid for the model.
|
903
|
-
"""
|
904
|
-
return cls.model_validate_json(safe_text_read(path))
|