fabricatio 0.2.10.dev1__cp312-cp312-win_amd64.whl → 0.2.11__cp312-cp312-win_amd64.whl

fabricatio/capabilities/rating.py

@@ -14,7 +14,7 @@ from fabricatio.models.generic import Display, ProposedAble
 from fabricatio.models.kwargs_types import CompositeScoreKwargs, ValidateKwargs
 from fabricatio.parser import JsonCapture
 from fabricatio.rust_instances import TEMPLATE_MANAGER
-from fabricatio.utils import fallback_kwargs, ok, override_kwargs
+from fabricatio.utils import ok, override_kwargs
 
 
 class Rating(Propose):
@@ -137,7 +137,7 @@ class Rating(Propose):
             or dict(zip(criteria, criteria, strict=True))
         )
 
-        return await self.rate_fine_grind(to_rate, manual, score_range, **fallback_kwargs(kwargs, co_extractor={}))
+        return await self.rate_fine_grind(to_rate, manual, score_range, **kwargs)
 
     async def draft_rating_manual(
         self, topic: str, criteria: Optional[Set[str]] = None, **kwargs: Unpack[ValidateKwargs[Dict[str, str]]]
@@ -338,6 +338,7 @@ class Rating(Propose):
         criteria: Optional[Set[str]] = None,
         weights: Optional[Dict[str, float]] = None,
         manual: Optional[Dict[str, str]] = None,
+        approx: bool = False,
         **kwargs: Unpack[ValidateKwargs[List[Dict[str, float]]]],
     ) -> List[float]:
         """Calculates the composite scores for a list of items based on a given topic and criteria.
@@ -348,6 +349,7 @@ class Rating(Propose):
             criteria (Optional[Set[str]]): A set of criteria for the rating. Defaults to None.
             weights (Optional[Dict[str, float]]): A dictionary of rating weights for each criterion. Defaults to None.
             manual (Optional[Dict[str, str]]): A dictionary of manual ratings for each item. Defaults to None.
+            approx (bool): Whether to use approximate rating criteria. Defaults to False.
             **kwargs (Unpack[ValidateKwargs]): Additional keyword arguments for the LLM usage.
 
         Returns:
@@ -355,6 +357,7 @@ class Rating(Propose):
         """
         criteria = ok(
             criteria
+            or (await self.draft_rating_criteria(topic, **override_kwargs(kwargs, default=None)) if approx else None)
             or await self.draft_rating_criteria_from_examples(topic, to_rate, **override_kwargs(kwargs, default=None))
         )
         weights = ok(

fabricatio/capabilities/task.py

@@ -3,7 +3,7 @@
 from types import CodeType
 from typing import Any, Dict, List, Optional, Tuple, Unpack
 
-import orjson
+import ujson
 
 from fabricatio.capabilities.propose import Propose
 from fabricatio.config import configs
@@ -20,9 +20,9 @@ class ProposeTask(Propose):
     """A class that proposes a task based on a prompt."""
 
     async def propose_task[T](
-        self,
-        prompt: str,
-        **kwargs: Unpack[ValidateKwargs[Task[T]]],
+            self,
+            prompt: str,
+            **kwargs: Unpack[ValidateKwargs[Task[T]]],
     ) -> Optional[Task[T]]:
         """Asynchronously proposes a task based on a given prompt and parameters.
 
@@ -44,11 +44,11 @@ class HandleTask(ToolBoxUsage):
     """A class that handles a task based on a task object."""
 
     async def draft_tool_usage_code(
-        self,
-        task: Task,
-        tools: List[Tool],
-        data: Dict[str, Any],
-        **kwargs: Unpack[ValidateKwargs],
+            self,
+            task: Task,
+            tools: List[Tool],
+            data: Dict[str, Any],
+            **kwargs: Unpack[ValidateKwargs],
     ) -> Optional[Tuple[CodeType, List[str]]]:
         """Asynchronously drafts the tool usage code for a task based on a given task object and tools."""
         logger.info(f"Drafting tool usage code for task: {task.briefing}")
@@ -60,7 +60,7 @@ class HandleTask(ToolBoxUsage):
 
         def _validator(response: str) -> Tuple[CodeType, List[str]] | None:
             if (source := PythonCapture.convert_with(response, lambda resp: compile(resp, "<string>", "exec"))) and (
-                to_extract := JsonCapture.convert_with(response, orjson.loads)
+                    to_extract := JsonCapture.convert_with(response, ujson.loads)
             ):
                 return source, to_extract
 
@@ -85,12 +85,12 @@ class HandleTask(ToolBoxUsage):
         )
 
     async def handle_fine_grind(
-        self,
-        task: Task,
-        data: Dict[str, Any],
-        box_choose_kwargs: Optional[ChooseKwargs] = None,
-        tool_choose_kwargs: Optional[ChooseKwargs] = None,
-        **kwargs: Unpack[ValidateKwargs],
+            self,
+            task: Task,
+            data: Dict[str, Any],
+            box_choose_kwargs: Optional[ChooseKwargs] = None,
+            tool_choose_kwargs: Optional[ChooseKwargs] = None,
+            **kwargs: Unpack[ValidateKwargs],
     ) -> Optional[Tuple]:
         """Asynchronously handles a task based on a given task object and parameters."""
         logger.info(f"Handling task: \n{task.briefing}")

fabricatio/config.py

@@ -86,8 +86,10 @@ class LLMConfig(BaseModel):
 
     tpm: Optional[PositiveInt] = Field(default=1000000)
     """The rate limit of the LLM model in tokens per minute. None means not checked."""
-
-
+    presence_penalty:Optional[PositiveFloat]=None
+    """The presence penalty of the LLM model."""
+    frequency_penalty:Optional[PositiveFloat]=None
+    """The frequency penalty of the LLM model."""
 class EmbeddingConfig(BaseModel):
     """Embedding configuration class."""
 
@@ -249,6 +251,11 @@ class TemplateConfig(BaseModel):
 
     rule_requirement_template: str = Field(default="rule_requirement")
     """The name of the rule requirement template which will be used to generate a rule requirement."""
+
+
+    extract_template: str = Field(default="extract")
+    """The name of the extract template which will be used to extract model from string."""
+
 class MagikaConfig(BaseModel):
     """Magika configuration class."""
 

fabricatio/decorators.py

@@ -6,14 +6,47 @@ from importlib.util import find_spec
 from inspect import signature
 from shutil import which
 from types import ModuleType
-from typing import Callable, List, Optional
-
-from questionary import confirm
+from typing import Callable, Coroutine, List, Optional
 
 from fabricatio.config import configs
 from fabricatio.journal import logger
 
 
+def precheck_package[**P, R](package_name: str, msg: str) -> Callable[[Callable[P, R]], Callable[P, R]]:
+    """Check if a package exists in the current environment.
+
+    Args:
+        package_name (str): The name of the package to check.
+        msg (str): The message to display if the package is not found.
+
+    Returns:
+        bool: True if the package exists, False otherwise.
+    """
+
+    def _wrapper(
+        func: Callable[P, R] | Callable[P, Coroutine[None, None, R]],
+    ) -> Callable[P, R] | Callable[P, Coroutine[None, None, R]]:
+        if iscoroutinefunction(func):
+
+            @wraps(func)
+            async def _async_inner(*args: P.args, **kwargs: P.kwargs) -> R:
+                if find_spec(package_name):
+                    return await func(*args, **kwargs)
+                raise RuntimeError(msg)
+
+            return _async_inner
+
+        @wraps(func)
+        def _inner(*args: P.args, **kwargs: P.kwargs) -> R:
+            if find_spec(package_name):
+                return func(*args, **kwargs)
+            raise RuntimeError(msg)
+
+        return _inner
+
+    return _wrapper
+
+
 def depend_on_external_cmd[**P, R](
     bin_name: str, install_tip: Optional[str], homepage: Optional[str] = None
 ) -> Callable[[Callable[P, R]], Callable[P, R]]:
@@ -68,6 +101,9 @@ def logging_execution_info[**P, R](func: Callable[P, R]) -> Callable[P, R]:
     return _wrapper
 
 
+@precheck_package(
+    "questionary", "'questionary' is required to run this function. Have you installed `fabricatio[qa]`?."
+)
 def confirm_to_execute[**P, R](func: Callable[P, R]) -> Callable[P, Optional[R]] | Callable[P, R]:
     """Decorator to confirm before executing a function.
 
@@ -80,6 +116,7 @@ def confirm_to_execute[**P, R](func: Callable[P, R]) -> Callable[P, Optional[R]]
     if not configs.general.confirm_on_ops:
         # Skip confirmation if the configuration is set to False
         return func
+    from questionary import confirm
 
     if iscoroutinefunction(func):
 
@@ -180,7 +217,9 @@ def use_temp_module[**P, R](modules: ModuleType | List[ModuleType]) -> Callable[
     return _decorator
 
 
-def logging_exec_time[**P, R](func: Callable[P, R]) -> Callable[P, R]:
+def logging_exec_time[**P, R](
+    func: Callable[P, R] | Callable[P, Coroutine[None, None, R]],
+) -> Callable[P, R] | Callable[P, Coroutine[None, None, R]]:
     """Decorator to log the execution time of a function.
 
     Args:
@@ -210,25 +249,3 @@ def logging_exec_time[**P, R](func: Callable[P, R]) -> Callable[P, R]:
         return result
 
     return _wrapper
-
-
-def precheck_package[**P, R](package_name: str, msg: str) -> Callable[[Callable[P, R]], Callable[P, R]]:
-    """Check if a package exists in the current environment.
-
-    Args:
-        package_name (str): The name of the package to check.
-        msg (str): The message to display if the package is not found.
-
-    Returns:
-        bool: True if the package exists, False otherwise.
-    """
-
-    def _wrapper(func: Callable[P, R]) -> Callable[P, R]:
-        def _inner(*args: P.args, **kwargs: P.kwargs) -> R:
-            if find_spec(package_name):
-                return func(*args, **kwargs)
-            raise RuntimeError(msg)
-
-        return _inner
-
-    return _wrapper

fabricatio/fs/__init__.py

@@ -1,5 +1,7 @@
 """FileSystem manipulation module for Fabricatio."""
+from importlib.util import find_spec
 
+from fabricatio.config import configs
 from fabricatio.fs.curd import (
     absolute_path,
     copy_file,
@@ -11,10 +13,9 @@ from fabricatio.fs.curd import (
     move_file,
     tree,
 )
-from fabricatio.fs.readers import MAGIKA, safe_json_read, safe_text_read
+from fabricatio.fs.readers import safe_json_read, safe_text_read
 
 __all__ = [
-    "MAGIKA",
     "absolute_path",
     "copy_file",
     "create_directory",
@@ -27,3 +28,9 @@ __all__ = [
     "safe_text_read",
     "tree",
 ]
+
+if find_spec("magika"):
+    from magika import Magika
+
+    MAGIKA = Magika(model_dir=configs.magika.model_dir)
+    __all__ += ["MAGIKA"]

fabricatio/fs/readers.py

@@ -1,17 +1,13 @@
 """Filesystem readers for Fabricatio."""
 
+import re
 from pathlib import Path
 from typing import Dict, List, Tuple
 
-import orjson
-import regex
-from magika import Magika
+import ujson
 
-from fabricatio.config import configs
 from fabricatio.journal import logger
 
-MAGIKA = Magika(model_dir=configs.magika.model_dir)
-
 
 def safe_text_read(path: Path | str) -> str:
     """Safely read the text from a file.
@@ -41,8 +37,8 @@ def safe_json_read(path: Path | str) -> Dict:
     """
     path = Path(path)
     try:
-        return orjson.loads(path.read_text(encoding="utf-8"))
-    except (orjson.JSONDecodeError, IsADirectoryError, FileNotFoundError) as e:
+        return ujson.loads(path.read_text(encoding="utf-8"))
+    except (ujson.JSONDecodeError, IsADirectoryError, FileNotFoundError) as e:
         logger.error(f"Failed to read file {path}: {e!s}")
         return {}
 
@@ -58,8 +54,8 @@ def extract_sections(string: str, level: int, section_char: str = "#") -> List[T
     Returns:
         List[Tuple[str, str]]: List of (header_text, section_content) tuples
     """
-    return regex.findall(
+    return re.findall(
         r"^%s{%d}\s+(.+?)\n((?:(?!^%s{%d}\s).|\n)*)" % (section_char, level, section_char, level),
         string,
-        regex.MULTILINE,
+        re.MULTILINE,
     )

fabricatio/models/action.py

@@ -12,12 +12,13 @@ Classes:
 import traceback
 from abc import abstractmethod
 from asyncio import Queue, create_task
-from typing import Any, Dict, Self, Tuple, Type, Union, final
+from typing import Any, Dict, Self, Sequence, Tuple, Type, Union, final
 
 from fabricatio.journal import logger
 from fabricatio.models.generic import WithBriefing
 from fabricatio.models.task import Task
 from fabricatio.models.usages import LLMUsage, ToolBoxUsage
+from fabricatio.utils import override_kwargs
 from pydantic import Field, PrivateAttr
 
 OUTPUT_KEY = "task_output"
@@ -55,7 +56,7 @@ class Action(WithBriefing, LLMUsage):
         self.description = self.description or self.__class__.__doc__ or ""
 
     @abstractmethod
-    async def _execute(self, *_:Any, **cxt) -> Any:
+    async def _execute(self, *_: Any, **cxt) -> Any:
         """Implement the core logic of the action.
 
         Args:
@@ -95,11 +96,12 @@ class Action(WithBriefing, LLMUsage):
             return f"## Your personality: \n{self.personality}\n# The action you are going to perform: \n{super().briefing}"
         return f"# The action you are going to perform: \n{super().briefing}"
 
-    def to_task_output(self)->Self:
+    def to_task_output(self, task_output_key: str = OUTPUT_KEY) -> Self:
         """Set the output key to OUTPUT_KEY and return the action instance."""
-        self.output_key=OUTPUT_KEY
+        self.output_key = task_output_key
         return self
 
+
 class WorkFlow(WithBriefing, ToolBoxUsage):
     """Manages sequences of actions to fulfill tasks.
 
@@ -121,9 +123,7 @@ class WorkFlow(WithBriefing, ToolBoxUsage):
     _instances: Tuple[Action, ...] = PrivateAttr(default_factory=tuple)
     """Instantiated action objects to be executed in this workflow."""
 
-    steps: Tuple[Union[Type[Action], Action], ...] = Field(
-        frozen=True,
-    )
+    steps: Sequence[Union[Type[Action], Action]] = Field(frozen=True)
     """The sequence of actions to be executed, can be action classes or instances."""
 
     task_input_key: str = Field(default=INPUT_KEY)
@@ -177,7 +177,7 @@ class WorkFlow(WithBriefing, ToolBoxUsage):
         current_action = None
         try:
             # Process each action in sequence
-            for i,step in enumerate(self._instances):
+            for i, step in enumerate(self._instances):
                 current_action = step.name
                 logger.info(f"Executing step [{i}] >> {current_action}")
 
@@ -227,8 +227,13 @@ class WorkFlow(WithBriefing, ToolBoxUsage):
             - Any extra_init_context values
         """
         logger.debug(f"Initializing context for workflow: {self.name}")
-        initial_context = {self.task_input_key: task, **dict(self.extra_init_context)}
-        await self._context.put(initial_context)
+        ctx = override_kwargs(self.extra_init_context, **task.extra_init_context)
+        if self.task_input_key in ctx:
+            raise ValueError(
+                f"Task input key: `{self.task_input_key}`, which is reserved, is already set in the init context"
+            )
+
+        await self._context.put({self.task_input_key: task, **ctx})
 
     def steps_fallback_to_self(self) -> Self:
         """Configure all steps to use this workflow's configuration as fallback.
@@ -245,7 +250,7 @@ class WorkFlow(WithBriefing, ToolBoxUsage):
         Returns:
             Self: The workflow instance for method chaining.
         """
-        self.provide_tools_to(i for i in self._instances if isinstance(i,ToolBoxUsage))
+        self.provide_tools_to(i for i in self._instances if isinstance(i, ToolBoxUsage))
         return self
 
     def update_init_context(self, /, **kwargs) -> Self:

fabricatio/models/extra/aricle_rag.py

@@ -1,22 +1,27 @@
 """A Module containing the article rag models."""
 
+import re
 from pathlib import Path
-from typing import ClassVar, Dict, List, Self, Unpack
+from typing import ClassVar, Dict, List, Optional, Self, Unpack
 
 from fabricatio.fs import safe_text_read
 from fabricatio.journal import logger
 from fabricatio.models.extra.rag import MilvusDataBase
 from fabricatio.models.generic import AsPrompt
 from fabricatio.models.kwargs_types import ChunkKwargs
-from fabricatio.rust import BibManager, split_into_chunks
-from fabricatio.utils import ok, wrapp_in_block
-from more_itertools.recipes import flatten
+from fabricatio.rust import BibManager, blake3_hash, is_chinese, split_into_chunks
+from fabricatio.utils import ok
+from more_itertools.recipes import flatten, unique
 from pydantic import Field
 
 
 class ArticleChunk(MilvusDataBase, AsPrompt):
     """The chunk of an article."""
 
+    etc_word: ClassVar[str] = "等"
+    and_word: ClassVar[str] = "与"
+    _cite_number: Optional[int] = None
+
     head_split: ClassVar[List[str]] = [
         "引 言",
         "引言",
@@ -48,12 +53,14 @@ class ArticleChunk(MilvusDataBase, AsPrompt):
 
     def _as_prompt_inner(self) -> Dict[str, str]:
         return {
-            self.article_title: f"{wrapp_in_block(self.chunk, 'Referring Content')}\n"
-            f"Authors: {';'.join(self.authors)}\n"
-            f"Published Year: {self.year}\n"
-            f"Bibtex Key: {self.bibtex_cite_key}\n",
+            f"[[{ok(self._cite_number, 'You need to update cite number first.')}]] reference `{self.article_title}` from {self.as_auther_seq()}": self.chunk
         }
 
+    @property
+    def cite_number(self) -> int:
+        """Get the cite number."""
+        return ok(self._cite_number, "cite number not set")
+
     def _prepare_vectorization_inner(self) -> str:
         return self.chunk
 
@@ -89,8 +96,9 @@ class ArticleChunk(MilvusDataBase, AsPrompt):
 
         result = [
             cls(chunk=c, year=year, authors=authors, article_title=article_title, bibtex_cite_key=key)
-            for c in split_into_chunks(cls.strip(safe_text_read(path)), **kwargs)
+            for c in split_into_chunks(cls.purge_numeric_citation(cls.strip(safe_text_read(path))), **kwargs)
         ]
+
         logger.debug(f"Number of chunks created from file {path.as_posix()}: {len(result)}")
         return result
 
@@ -118,3 +126,129 @@ class ArticleChunk(MilvusDataBase, AsPrompt):
             logger.warning("No decrease at tail strip, which is might be abnormal.")
 
         return string
+
+    def as_typst_cite(self) -> str:
+        """As typst cite."""
+        return f"#cite(<{self.bibtex_cite_key}>)"
+
+    @staticmethod
+    def purge_numeric_citation(string: str) -> str:
+        """Purge numeric citation."""
+        import re
+
+        return re.sub(r"\[[\d\s,\\~–-]+]", "", string)
+
+    @property
+    def auther_firstnames(self) -> List[str]:
+        """Get the first name of the authors."""
+        ret = []
+        for n in self.authors:
+            if is_chinese(n):
+                ret.append(n[0])
+            else:
+                ret.append(n.split()[-1])
+        return ret
+
+    def as_auther_seq(self) -> str:
+        """Get the auther sequence."""
+        match len(self.authors):
+            case 0:
+                raise ValueError("No authors found")
+            case 1:
+                return f"（{self.auther_firstnames[0]}，{self.year}）{self.as_typst_cite()}"
+            case 2:
+                return f"（{self.auther_firstnames[0]}{self.and_word}{self.auther_firstnames[1]}，{self.year}）{self.as_typst_cite()}"
+            case 3:
+                return f"（{self.auther_firstnames[0]}，{self.auther_firstnames[1]}{self.and_word}{self.auther_firstnames[2]}，{self.year}）{self.as_typst_cite()}"
+            case _:
+                return f"（{self.auther_firstnames[0]}，{self.auther_firstnames[1]}{self.and_word}{self.auther_firstnames[2]}{self.etc_word}，{self.year}）{self.as_typst_cite()}"
+
+    def update_cite_number(self, cite_number: int) -> Self:
+        """Update the cite number."""
+        self._cite_number = cite_number
+        return self
+
+
+class CitationManager(AsPrompt):
+    """Citation manager."""
+
+    article_chunks: List[ArticleChunk] = Field(default_factory=list)
+    """Article chunks."""
+
+    pat: str = r"(\[\[([\d\s,-]*)]])"
+    """Regex pattern to match citations."""
+    sep: str = ","
+    """Separator for citation numbers."""
+    abbr_sep: str = "-"
+    """Separator for abbreviated citation numbers."""
+
+    def update_chunks(
+        self, article_chunks: List[ArticleChunk], set_cite_number: bool = True, dedup: bool = True
+    ) -> Self:
+        """Update article chunks."""
+        self.article_chunks.clear()
+        self.article_chunks.extend(article_chunks)
+        if dedup:
+            self.article_chunks = list(unique(self.article_chunks, lambda c: blake3_hash(c.chunk.encode())))
+        if set_cite_number:
+            self.set_cite_number_all()
+        return self
+
+    def empty(self) -> Self:
+        """Empty the article chunks."""
+        self.article_chunks.clear()
+        return self
+
+    def add_chunks(self, article_chunks: List[ArticleChunk], set_cite_number: bool = True, dedup: bool = True) -> Self:
+        """Add article chunks."""
+        self.article_chunks.extend(article_chunks)
+        if dedup:
+            self.article_chunks = list(unique(self.article_chunks, lambda c: blake3_hash(c.chunk.encode())))
+        if set_cite_number:
+            self.set_cite_number_all()
+        return self
+
+    def set_cite_number_all(self) -> Self:
+        """Set citation numbers for all article chunks."""
+        for i, a in enumerate(self.article_chunks, 1):
+            a.update_cite_number(i)
+        return self
+
+    def _as_prompt_inner(self) -> Dict[str, str]:
+        """Generate prompt inner representation."""
+        return {"References": "\n".join(r.as_prompt() for r in self.article_chunks)}
+
+    def apply(self, string: str) -> str:
+        """Apply citation replacements to the input string."""
+        for origin, m in re.findall(self.pat, string):
+            logger.info(f"Matching citation: {m}")
+            notations = self.convert_to_numeric_notations(m)
+            logger.info(f"Citing Notations: {notations}")
+            citation_number_seq = list(flatten(self.decode_expr(n) for n in notations))
+            logger.info(f"Citation Number Sequence: {citation_number_seq}")
+            dedup = self.deduplicate_citation(citation_number_seq)
+            logger.info(f"Deduplicated Citation Number Sequence: {dedup}")
+            string = string.replace(origin, self.unpack_cite_seq(dedup))
+        return string
+
+    def decode_expr(self, string: str) -> List[int]:
+        """Decode citation expression into a list of integers."""
+        if self.abbr_sep in string:
+            start, end = string.split(self.abbr_sep)
+            return list(range(int(start), int(end) + 1))
+        return [int(string)]
+
+    def convert_to_numeric_notations(self, string: str) -> List[str]:
+        """Convert citation string into numeric notations."""
+        return [s.strip() for s in string.split(self.sep)]
+
+    def deduplicate_citation(self, citation_seq: List[int]) -> List[int]:
+        """Deduplicate citation sequence."""
+        chunk_seq = [a for a in self.article_chunks if a.cite_number in citation_seq]
+        deduped = unique(chunk_seq, lambda a: a.bibtex_cite_key)
+        return [a.cite_number for a in deduped]
+
+    def unpack_cite_seq(self, citation_seq: List[int]) -> str:
+        """Unpack citation sequence into a string."""
+        chunk_seq = [a for a in self.article_chunks if a.cite_number in citation_seq]
+        return "".join(a.as_typst_cite() for a in chunk_seq)