fabricatio 0.2.10.dev0__cp312-cp312-win_amd64.whl → 0.2.11__cp312-cp312-win_amd64.whl

fabricatio/actions/rules.py

@@ -1,15 +1,16 @@
 """A module containing the DraftRuleSet action."""
 
-from typing import List, Optional
+from typing import Any, List, Mapping, Optional, Self, Tuple
 
 from fabricatio.capabilities.check import Check
 from fabricatio.journal import logger
 from fabricatio.models.action import Action
 from fabricatio.models.extra.rule import RuleSet
+from fabricatio.models.generic import FromMapping
 from fabricatio.utils import ok
 
 
-class DraftRuleSet(Action, Check):
+class DraftRuleSet(Action, Check, FromMapping):
     """Action to draft a ruleset based on a given requirement description."""
 
     output_key: str = "drafted_ruleset"
@@ -45,8 +46,13 @@ class DraftRuleSet(Action, Check):
             logger.warning(f"Drafting Rule Failed for:\n{ruleset_requirement}")
         return ruleset
 
+    @classmethod
+    def from_mapping(cls, mapping: Mapping[str, Tuple[int, str]], **kwargs) -> List[Self]:
+        """Create a list of DraftRuleSet actions from a mapping of output keys to tuples of rule counts and requirements."""
+        return [cls(ruleset_requirement=r, rule_count=c, output_key=k, **kwargs) for k, (c, r) in mapping.items()]
 
-class GatherRuleset(Action):
+
+class GatherRuleset(Action, FromMapping):
     """Action to gather a ruleset from a given requirement description."""
 
     output_key: str = "gathered_ruleset"
@@ -55,6 +61,11 @@ class GatherRuleset(Action):
     to_gather: List[str]
     """the cxt name of RuleSet to gather"""
 
+    @classmethod
+    def from_mapping(cls, mapping: Mapping[str, List[str]], **kwargs: Any) -> List[Self]:
+        """Create a list of GatherRuleset actions from a mapping of output keys to tuples of rule counts and requirements."""
+        return [cls(to_gather=t, output_key=k, **kwargs) for k, t in mapping.items()]
+
     async def _execute(self, **cxt) -> RuleSet:
         logger.info(f"Gathering Ruleset from {self.to_gather}")
         # Fix for not_found

fabricatio/capabilities/extract.py

@@ -0,0 +1,70 @@
+"""A module that provide capabilities for extracting information from a given source to a model."""
+
+from typing import List, Optional, Type, Unpack, overload
+
+from fabricatio import TEMPLATE_MANAGER
+from fabricatio.capabilities.propose import Propose
+from fabricatio.config import configs
+from fabricatio.models.generic import ProposedAble
+from fabricatio.models.kwargs_types import ValidateKwargs
+
+
+class Extract(Propose):
+    """A class that extract information from a given source to a model."""
+
+    @overload
+    async def extract[M: ProposedAble](
+        self,
+        cls: Type[M],
+        source: str,
+        extract_requirement: Optional[str] = None,
+        align_language: bool = True,
+        **kwargs: Unpack[ValidateKwargs[M]],
+    ) -> M: ...
+    @overload
+    async def extract[M: ProposedAble](
+        self,
+        cls: Type[M],
+        source: str,
+        extract_requirement: Optional[str] = None,
+        align_language: bool = True,
+        **kwargs: Unpack[ValidateKwargs[None]],
+    ) -> Optional[M]: ...
+
+    @overload
+    async def extract[M: ProposedAble](
+        self,
+        cls: Type[M],
+        source: List[str],
+        extract_requirement: Optional[str] = None,
+        align_language: bool = True,
+        **kwargs: Unpack[ValidateKwargs[M]],
+    ) -> List[M]: ...
+    @overload
+    async def extract[M: ProposedAble](
+        self,
+        cls: Type[M],
+        source: List[str],
+        extract_requirement: Optional[str] = None,
+        align_language: bool = True,
+        **kwargs: Unpack[ValidateKwargs[None]],
+    ) -> List[Optional[M]]: ...
+    async def extract[M: ProposedAble](
+        self,
+        cls: Type[M],
+        source: List[str] | str,
+        extract_requirement: Optional[str] = None,
+        align_language: bool = True,
+        **kwargs: Unpack[ValidateKwargs[Optional[M]]],
+    ) -> M | List[M] | Optional[M] | List[Optional[M]]:
+        """Extract information from a given source to a model."""
+        return await self.propose(
+            cls,
+            prompt=TEMPLATE_MANAGER.render_template(
+                configs.templates.extract_template,
+                [{"source": s, "extract_requirement": extract_requirement} for s in source]
+                if isinstance(source, list)
+                else {"source": source, "extract_requirement": extract_requirement, "align_language": align_language},
+            ),
+            **kwargs,
+        )

fabricatio/capabilities/rag.py

@@ -130,7 +130,7 @@ class RAG(EmbeddingUsage):
         if isinstance(data, MilvusDataBase):
             data = [data]
 
-        data_vec = await self.vectorize([d.to_vectorize for d in data])
+        data_vec = await self.vectorize([d.prepare_vectorization() for d in data])
         prepared_data = [d.prepare_insertion(vec) for d, vec in zip(data, data_vec, strict=True)]
 
         c_name = collection_name or self.safe_target_collection
@@ -188,13 +188,15 @@ class RAG(EmbeddingUsage):
     async def aretrieve[D: MilvusDataBase](
         self,
         query: List[str] | str,
+        document_model: Type[D],
         final_limit: int = 20,
-        **kwargs: Unpack[FetchKwargs[D]],
+        **kwargs: Unpack[FetchKwargs],
     ) -> List[D]:
         """Retrieve data from the collection.
 
         Args:
             query (List[str] | str): The query to be used for retrieval.
+            document_model (Type[D]): The model class used to convert retrieved data into document objects.
             final_limit (int): The final limit on the number of results to return.
             **kwargs (Unpack[FetchKwargs]): Additional keyword arguments for retrieval.
 
@@ -206,6 +208,7 @@ class RAG(EmbeddingUsage):
         return (
             await self.afetch_document(
                 vecs=(await self.vectorize(query)),
+                document_model=document_model,
                 **kwargs,
             )
         )[:final_limit]

fabricatio/capabilities/rating.py

@@ -14,7 +14,7 @@ from fabricatio.models.generic import Display, ProposedAble
 from fabricatio.models.kwargs_types import CompositeScoreKwargs, ValidateKwargs
 from fabricatio.parser import JsonCapture
 from fabricatio.rust_instances import TEMPLATE_MANAGER
-from fabricatio.utils import fallback_kwargs, ok, override_kwargs
+from fabricatio.utils import ok, override_kwargs
 
 
 class Rating(Propose):
@@ -137,7 +137,7 @@ class Rating(Propose):
             or dict(zip(criteria, criteria, strict=True))
         )
 
-        return await self.rate_fine_grind(to_rate, manual, score_range, **fallback_kwargs(kwargs, co_extractor={}))
+        return await self.rate_fine_grind(to_rate, manual, score_range, **kwargs)
 
     async def draft_rating_manual(
         self, topic: str, criteria: Optional[Set[str]] = None, **kwargs: Unpack[ValidateKwargs[Dict[str, str]]]
@@ -338,6 +338,7 @@ class Rating(Propose):
         criteria: Optional[Set[str]] = None,
         weights: Optional[Dict[str, float]] = None,
         manual: Optional[Dict[str, str]] = None,
+        approx: bool = False,
         **kwargs: Unpack[ValidateKwargs[List[Dict[str, float]]]],
     ) -> List[float]:
         """Calculates the composite scores for a list of items based on a given topic and criteria.
@@ -348,6 +349,7 @@ class Rating(Propose):
             criteria (Optional[Set[str]]): A set of criteria for the rating. Defaults to None.
             weights (Optional[Dict[str, float]]): A dictionary of rating weights for each criterion. Defaults to None.
             manual (Optional[Dict[str, str]]): A dictionary of manual ratings for each item. Defaults to None.
+            approx (bool): Whether to use approximate rating criteria. Defaults to False.
             **kwargs (Unpack[ValidateKwargs]): Additional keyword arguments for the LLM usage.
 
         Returns:
@@ -355,6 +357,7 @@ class Rating(Propose):
         """
         criteria = ok(
             criteria
+            or (await self.draft_rating_criteria(topic, **override_kwargs(kwargs, default=None)) if approx else None)
             or await self.draft_rating_criteria_from_examples(topic, to_rate, **override_kwargs(kwargs, default=None))
         )
         weights = ok(

fabricatio/capabilities/task.py

@@ -3,7 +3,7 @@
 from types import CodeType
 from typing import Any, Dict, List, Optional, Tuple, Unpack
 
-import orjson
+import ujson
 
 from fabricatio.capabilities.propose import Propose
 from fabricatio.config import configs
@@ -20,9 +20,9 @@ class ProposeTask(Propose):
     """A class that proposes a task based on a prompt."""
 
     async def propose_task[T](
-        self,
-        prompt: str,
-        **kwargs: Unpack[ValidateKwargs[Task[T]]],
+            self,
+            prompt: str,
+            **kwargs: Unpack[ValidateKwargs[Task[T]]],
     ) -> Optional[Task[T]]:
         """Asynchronously proposes a task based on a given prompt and parameters.
 
@@ -44,11 +44,11 @@ class HandleTask(ToolBoxUsage):
     """A class that handles a task based on a task object."""
 
     async def draft_tool_usage_code(
-        self,
-        task: Task,
-        tools: List[Tool],
-        data: Dict[str, Any],
-        **kwargs: Unpack[ValidateKwargs],
+            self,
+            task: Task,
+            tools: List[Tool],
+            data: Dict[str, Any],
+            **kwargs: Unpack[ValidateKwargs],
     ) -> Optional[Tuple[CodeType, List[str]]]:
         """Asynchronously drafts the tool usage code for a task based on a given task object and tools."""
         logger.info(f"Drafting tool usage code for task: {task.briefing}")
@@ -60,7 +60,7 @@ class HandleTask(ToolBoxUsage):
 
         def _validator(response: str) -> Tuple[CodeType, List[str]] | None:
             if (source := PythonCapture.convert_with(response, lambda resp: compile(resp, "<string>", "exec"))) and (
-                to_extract := JsonCapture.convert_with(response, orjson.loads)
+                    to_extract := JsonCapture.convert_with(response, ujson.loads)
             ):
                 return source, to_extract
 
@@ -85,12 +85,12 @@ class HandleTask(ToolBoxUsage):
         )
 
     async def handle_fine_grind(
-        self,
-        task: Task,
-        data: Dict[str, Any],
-        box_choose_kwargs: Optional[ChooseKwargs] = None,
-        tool_choose_kwargs: Optional[ChooseKwargs] = None,
-        **kwargs: Unpack[ValidateKwargs],
+            self,
+            task: Task,
+            data: Dict[str, Any],
+            box_choose_kwargs: Optional[ChooseKwargs] = None,
+            tool_choose_kwargs: Optional[ChooseKwargs] = None,
+            **kwargs: Unpack[ValidateKwargs],
     ) -> Optional[Tuple]:
         """Asynchronously handles a task based on a given task object and parameters."""
         logger.info(f"Handling task: \n{task.briefing}")

fabricatio/config.py

@@ -86,8 +86,10 @@ class LLMConfig(BaseModel):
 
     tpm: Optional[PositiveInt] = Field(default=1000000)
     """The rate limit of the LLM model in tokens per minute. None means not checked."""
-
-
+    presence_penalty:Optional[PositiveFloat]=None
+    """The presence penalty of the LLM model."""
+    frequency_penalty:Optional[PositiveFloat]=None
+    """The frequency penalty of the LLM model."""
 class EmbeddingConfig(BaseModel):
     """Embedding configuration class."""
 
@@ -249,6 +251,11 @@ class TemplateConfig(BaseModel):
 
     rule_requirement_template: str = Field(default="rule_requirement")
     """The name of the rule requirement template which will be used to generate a rule requirement."""
+
+
+    extract_template: str = Field(default="extract")
+    """The name of the extract template which will be used to extract model from string."""
+
 class MagikaConfig(BaseModel):
     """Magika configuration class."""
 

fabricatio/decorators.py

@@ -6,14 +6,47 @@ from importlib.util import find_spec
 from inspect import signature
 from shutil import which
 from types import ModuleType
-from typing import Callable, List, Optional
-
-from questionary import confirm
+from typing import Callable, Coroutine, List, Optional
 
 from fabricatio.config import configs
 from fabricatio.journal import logger
 
 
+def precheck_package[**P, R](package_name: str, msg: str) -> Callable[[Callable[P, R]], Callable[P, R]]:
+    """Check if a package exists in the current environment.
+
+    Args:
+        package_name (str): The name of the package to check.
+        msg (str): The message to display if the package is not found.
+
+    Returns:
+        bool: True if the package exists, False otherwise.
+    """
+
+    def _wrapper(
+        func: Callable[P, R] | Callable[P, Coroutine[None, None, R]],
+    ) -> Callable[P, R] | Callable[P, Coroutine[None, None, R]]:
+        if iscoroutinefunction(func):
+
+            @wraps(func)
+            async def _async_inner(*args: P.args, **kwargs: P.kwargs) -> R:
+                if find_spec(package_name):
+                    return await func(*args, **kwargs)
+                raise RuntimeError(msg)
+
+            return _async_inner
+
+        @wraps(func)
+        def _inner(*args: P.args, **kwargs: P.kwargs) -> R:
+            if find_spec(package_name):
+                return func(*args, **kwargs)
+            raise RuntimeError(msg)
+
+        return _inner
+
+    return _wrapper
+
+
 def depend_on_external_cmd[**P, R](
     bin_name: str, install_tip: Optional[str], homepage: Optional[str] = None
 ) -> Callable[[Callable[P, R]], Callable[P, R]]:
@@ -68,6 +101,9 @@ def logging_execution_info[**P, R](func: Callable[P, R]) -> Callable[P, R]:
     return _wrapper
 
 
+@precheck_package(
+    "questionary", "'questionary' is required to run this function. Have you installed `fabricatio[qa]`?."
+)
 def confirm_to_execute[**P, R](func: Callable[P, R]) -> Callable[P, Optional[R]] | Callable[P, R]:
     """Decorator to confirm before executing a function.
 
@@ -80,6 +116,7 @@ def confirm_to_execute[**P, R](func: Callable[P, R]) -> Callable[P, Optional[R]]
     if not configs.general.confirm_on_ops:
         # Skip confirmation if the configuration is set to False
         return func
+    from questionary import confirm
 
     if iscoroutinefunction(func):
 
@@ -180,7 +217,9 @@ def use_temp_module[**P, R](modules: ModuleType | List[ModuleType]) -> Callable[
     return _decorator
 
 
-def logging_exec_time[**P, R](func: Callable[P, R]) -> Callable[P, R]:
+def logging_exec_time[**P, R](
+    func: Callable[P, R] | Callable[P, Coroutine[None, None, R]],
+) -> Callable[P, R] | Callable[P, Coroutine[None, None, R]]:
     """Decorator to log the execution time of a function.
 
     Args:
@@ -210,25 +249,3 @@ def logging_exec_time[**P, R](func: Callable[P, R]) -> Callable[P, R]:
         return result
 
     return _wrapper
-
-
-def precheck_package[**P, R](package_name: str, msg: str) -> Callable[[Callable[P, R]], Callable[P, R]]:
-    """Check if a package exists in the current environment.
-
-    Args:
-        package_name (str): The name of the package to check.
-        msg (str): The message to display if the package is not found.
-
-    Returns:
-        bool: True if the package exists, False otherwise.
-    """
-
-    def _wrapper(func: Callable[P, R]) -> Callable[P, R]:
-        def _inner(*args: P.args, **kwargs: P.kwargs) -> R:
-            if find_spec(package_name):
-                return func(*args, **kwargs)
-            raise RuntimeError(msg)
-
-        return _inner
-
-    return _wrapper

fabricatio/fs/__init__.py

@@ -1,5 +1,7 @@
 """FileSystem manipulation module for Fabricatio."""
+from importlib.util import find_spec
 
+from fabricatio.config import configs
 from fabricatio.fs.curd import (
     absolute_path,
     copy_file,
@@ -11,10 +13,9 @@ from fabricatio.fs.curd import (
     move_file,
     tree,
 )
-from fabricatio.fs.readers import MAGIKA, safe_json_read, safe_text_read
+from fabricatio.fs.readers import safe_json_read, safe_text_read
 
 __all__ = [
-    "MAGIKA",
     "absolute_path",
     "copy_file",
     "create_directory",
@@ -27,3 +28,9 @@ __all__ = [
     "safe_text_read",
     "tree",
 ]
+
+if find_spec("magika"):
+    from magika import Magika
+
+    MAGIKA = Magika(model_dir=configs.magika.model_dir)
+    __all__ += ["MAGIKA"]

fabricatio/fs/readers.py

@@ -1,17 +1,13 @@
 """Filesystem readers for Fabricatio."""
 
+import re
 from pathlib import Path
 from typing import Dict, List, Tuple
 
-import orjson
-import regex
-from magika import Magika
+import ujson
 
-from fabricatio.config import configs
 from fabricatio.journal import logger
 
-MAGIKA = Magika(model_dir=configs.magika.model_dir)
-
 
 def safe_text_read(path: Path | str) -> str:
     """Safely read the text from a file.
@@ -41,8 +37,8 @@ def safe_json_read(path: Path | str) -> Dict:
     """
     path = Path(path)
     try:
-        return orjson.loads(path.read_text(encoding="utf-8"))
-    except (orjson.JSONDecodeError, IsADirectoryError, FileNotFoundError) as e:
+        return ujson.loads(path.read_text(encoding="utf-8"))
+    except (ujson.JSONDecodeError, IsADirectoryError, FileNotFoundError) as e:
         logger.error(f"Failed to read file {path}: {e!s}")
         return {}
 
@@ -58,8 +54,8 @@ def extract_sections(string: str, level: int, section_char: str = "#") -> List[T
     Returns:
         List[Tuple[str, str]]: List of (header_text, section_content) tuples
     """
-    return regex.findall(
+    return re.findall(
         r"^%s{%d}\s+(.+?)\n((?:(?!^%s{%d}\s).|\n)*)" % (section_char, level, section_char, level),
         string,
-        regex.MULTILINE,
+        re.MULTILINE,
     )

fabricatio/models/action.py

@@ -12,12 +12,13 @@ Classes:
 import traceback
 from abc import abstractmethod
 from asyncio import Queue, create_task
-from typing import Any, Dict, Self, Tuple, Type, Union, final
+from typing import Any, Dict, Self, Sequence, Tuple, Type, Union, final
 
 from fabricatio.journal import logger
 from fabricatio.models.generic import WithBriefing
 from fabricatio.models.task import Task
 from fabricatio.models.usages import LLMUsage, ToolBoxUsage
+from fabricatio.utils import override_kwargs
 from pydantic import Field, PrivateAttr
 
 OUTPUT_KEY = "task_output"
@@ -55,7 +56,7 @@ class Action(WithBriefing, LLMUsage):
         self.description = self.description or self.__class__.__doc__ or ""
 
     @abstractmethod
-    async def _execute(self, *_:Any, **cxt) -> Any:
+    async def _execute(self, *_: Any, **cxt) -> Any:
         """Implement the core logic of the action.
 
         Args:
@@ -95,11 +96,12 @@ class Action(WithBriefing, LLMUsage):
             return f"## Your personality: \n{self.personality}\n# The action you are going to perform: \n{super().briefing}"
         return f"# The action you are going to perform: \n{super().briefing}"
 
-    def to_task_output(self)->Self:
+    def to_task_output(self, task_output_key: str = OUTPUT_KEY) -> Self:
         """Set the output key to OUTPUT_KEY and return the action instance."""
-        self.output_key=OUTPUT_KEY
+        self.output_key = task_output_key
         return self
 
+
 class WorkFlow(WithBriefing, ToolBoxUsage):
     """Manages sequences of actions to fulfill tasks.
 
@@ -121,9 +123,7 @@ class WorkFlow(WithBriefing, ToolBoxUsage):
     _instances: Tuple[Action, ...] = PrivateAttr(default_factory=tuple)
     """Instantiated action objects to be executed in this workflow."""
 
-    steps: Tuple[Union[Type[Action], Action], ...] = Field(
-        frozen=True,
-    )
+    steps: Sequence[Union[Type[Action], Action]] = Field(frozen=True)
     """The sequence of actions to be executed, can be action classes or instances."""
 
     task_input_key: str = Field(default=INPUT_KEY)
@@ -177,7 +177,7 @@ class WorkFlow(WithBriefing, ToolBoxUsage):
         current_action = None
         try:
             # Process each action in sequence
-            for i,step in enumerate(self._instances):
+            for i, step in enumerate(self._instances):
                 current_action = step.name
                 logger.info(f"Executing step [{i}] >> {current_action}")
 
@@ -227,8 +227,13 @@ class WorkFlow(WithBriefing, ToolBoxUsage):
             - Any extra_init_context values
         """
         logger.debug(f"Initializing context for workflow: {self.name}")
-        initial_context = {self.task_input_key: task, **dict(self.extra_init_context)}
-        await self._context.put(initial_context)
+        ctx = override_kwargs(self.extra_init_context, **task.extra_init_context)
+        if self.task_input_key in ctx:
+            raise ValueError(
+                f"Task input key: `{self.task_input_key}`, which is reserved, is already set in the init context"
+            )
+
+        await self._context.put({self.task_input_key: task, **ctx})
 
     def steps_fallback_to_self(self) -> Self:
         """Configure all steps to use this workflow's configuration as fallback.
@@ -245,7 +250,7 @@ class WorkFlow(WithBriefing, ToolBoxUsage):
         Returns:
             Self: The workflow instance for method chaining.
         """
-        self.provide_tools_to(i for i in self._instances if isinstance(i,ToolBoxUsage))
+        self.provide_tools_to(i for i in self._instances if isinstance(i, ToolBoxUsage))
         return self
 
     def update_init_context(self, /, **kwargs) -> Self:

fabricatio/models/adv_kwargs_types.py

@@ -1,10 +1,9 @@
 """A module containing kwargs types for content correction and checking operations."""
 
 from importlib.util import find_spec
-from typing import Required, Type, TypedDict
+from typing import NotRequired, TypedDict
 
 from fabricatio.models.extra.problem import Improvement
-from fabricatio.models.extra.rag import MilvusDataBase
 from fabricatio.models.extra.rule import RuleSet
 from fabricatio.models.generic import SketchedAble
 from fabricatio.models.kwargs_types import ReferencedKwargs
@@ -49,19 +48,13 @@ if find_spec("pymilvus"):
         schema: CollectionSchema | None
         index_params: IndexParams | None
 
-    class FetchKwargs[D: MilvusDataBase](TypedDict, total=False):
+    class FetchKwargs(TypedDict):
         """Arguments for fetching data from vector collections.
 
         Controls how data is retrieved from vector databases, including filtering
         and result limiting parameters.
         """
 
-        document_model: Required[Type[D]]
-        collection_name: str | None
-        similarity_threshold: float
-        result_per_query: int
-
-    class RetrievalKwargs(FetchKwargs, total=False):
-        """Arguments for retrieval operations."""
-
-        final_limit: int
+        collection_name: NotRequired[str | None]
+        similarity_threshold: NotRequired[float]
+        result_per_query: NotRequired[int]