fabricatio 0.2.4.dev3__tar.gz → 0.2.5.dev1__tar.gz

{fabricatio-0.2.4.dev3 → fabricatio-0.2.5.dev1}/Cargo.lock

@@ -112,6 +112,19 @@ version = "0.22.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
 
+[[package]]
+name = "biblatex"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a35a7317fcbdbef94b60d0dd0a658711a936accfce4a631fea4bf8e527eff3c2"
+dependencies = [
+ "numerals",
+ "paste",
+ "strum",
+ "unicode-normalization",
+ "unscanny",
+]
+
 [[package]]
 name = "bitflags"
 version = "2.8.0"
@@ -254,6 +267,31 @@ dependencies = [
  "cfg-if",
 ]
 
+[[package]]
+name = "crossbeam-deque"
+version = "0.8.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51"
+dependencies = [
+ "crossbeam-epoch",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-epoch"
+version = "0.9.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
+dependencies = [
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
+
 [[package]]
 name = "crypto-common"
 version = "0.1.6"
@@ -372,6 +410,12 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "either"
+version = "1.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
+
 [[package]]
 name = "encoding_rs"
 version = "0.8.35"
@@ -401,10 +445,12 @@ dependencies = [
 name = "fabricatio"
 version = "0.0.0"
 dependencies = [
+ "biblatex",
  "blake3",
  "handlebars",
  "pyo3",
  "pythonize",
+ "rayon",
  "serde_json",
  "walkdir",
 ]
@@ -1021,6 +1067,12 @@ dependencies = [
  "num-modular",
 ]
 
+[[package]]
+name = "numerals"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e25be21376a772d15f97ae789845340a9651d3c4246ff5ebb6a2b35f9c37bd31"
+
 [[package]]
 name = "object"
 version = "0.36.7"
@@ -1086,6 +1138,12 @@ version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d"
 
+[[package]]
+name = "paste"
+version = "1.0.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
+
 [[package]]
 name = "percent-encoding"
 version = "2.3.1"
@@ -1252,6 +1310,26 @@ dependencies = [
  "proc-macro2",
 ]
 
+[[package]]
+name = "rayon"
+version = "1.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa"
+dependencies = [
+ "either",
+ "rayon-core",
+]
+
+[[package]]
+name = "rayon-core"
+version = "1.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2"
+dependencies = [
+ "crossbeam-deque",
+ "crossbeam-utils",
+]
+
 [[package]]
 name = "redox_syscall"
 version = "0.5.8"
@@ -1540,6 +1618,28 @@ version = "0.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
 
+[[package]]
+name = "strum"
+version = "0.26.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06"
+dependencies = [
+ "strum_macros",
+]
+
+[[package]]
+name = "strum_macros"
+version = "0.26.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be"
+dependencies = [
+ "heck",
+ "proc-macro2",
+ "quote",
+ "rustversion",
+ "syn",
+]
+
 [[package]]
 name = "subtle"
 version = "2.6.1"
@@ -1659,6 +1759,21 @@ dependencies = [
  "zerovec",
 ]
 
+[[package]]
+name = "tinyvec"
+version = "1.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09b3661f17e86524eccd4371ab0429194e0d7c008abb45f7a7495b1719463c71"
+dependencies = [
+ "tinyvec_macros",
+]
+
+[[package]]
+name = "tinyvec_macros"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
+
 [[package]]
 name = "tokio"
 version = "1.43.0"
@@ -1789,12 +1904,27 @@ version = "1.0.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a210d160f08b701c8721ba1c726c11662f877ea6b7094007e1ca9a1041945034"
 
+[[package]]
+name = "unicode-normalization"
+version = "0.1.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956"
+dependencies = [
+ "tinyvec",
+]
+
 [[package]]
 name = "unindent"
 version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce"
 
+[[package]]
+name = "unscanny"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e9df2af067a7953e9c3831320f35c1cc0600c30d44d9f7a12b01db1cd88d6b47"
+
 [[package]]
 name = "untrusted"
 version = "0.9.0"

{fabricatio-0.2.4.dev3 → fabricatio-0.2.5.dev1}/Cargo.toml

@@ -6,10 +6,12 @@ edition = "2024"
 crate-type = ["cdylib"]
 
 [dependencies]
+biblatex = "0.10.0"
 blake3 = "1.6.0"
 handlebars = "6.3.1"
 pyo3 = { version = "0.23.4", features = ["extension-module"] }
 pythonize = "0.23.0"
+rayon = "1.10.0"
 serde_json = "1.0.138"
 walkdir = "2.5.0"
 

{fabricatio-0.2.4.dev3 → fabricatio-0.2.5.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fabricatio
-Version: 0.2.4.dev3
+Version: 0.2.5.dev1
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: 3.12
@@ -9,6 +9,7 @@ Classifier: Framework :: AsyncIO
 Classifier: Framework :: Pydantic :: 2
 Classifier: Typing :: Typed
 Requires-Dist: appdirs>=1.4.4
+Requires-Dist: async-cache>=1.1.1
 Requires-Dist: asyncio>=3.4.3
 Requires-Dist: asyncstdlib>=3.13.0
 Requires-Dist: litellm>=1.60.0

fabricatio-0.2.5.dev1/examples/write_outline/.gitignore

@@ -0,0 +1,2 @@
+article_briefing.txt
+out.typ

fabricatio-0.2.5.dev1/examples/write_outline/write_outline.py

@@ -0,0 +1,24 @@
+"""Example of using the library."""
+
+import asyncio
+
+from fabricatio import Event, Role, WriteOutlineWorkFlow, logger
+
+
+async def main() -> None:
+    """Main function."""
+    role = Role(
+        name="Undergraduate Researcher",
+        description="Write an outline for an article in typst format.",
+        registry={Event.quick_instantiate(ns := "article"): WriteOutlineWorkFlow},
+    )
+
+    proposed_task = await role.propose_task(
+        "You need to read the `./article_briefing.txt` file and write an outline for the article in typst format. The outline should be saved in the `./out.typ` file.",
+    )
+    path = await proposed_task.delegate(ns)
+    logger.success(f"The outline is saved in:\n{path}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

{fabricatio-0.2.4.dev3 → fabricatio-0.2.5.dev1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "fabricatio"
-version = "0.2.4-dev3"
+version = "0.2.5-dev1"
 description = "A LLM multi-agent framework."
 readme = "README.md"
 license = { file = "LICENSE" }
@@ -28,6 +28,7 @@ keywords = [
 requires-python = ">=3.12,<3.13"
 dependencies = [
     "appdirs>=1.4.4",
+    "async-cache>=1.1.1",
     "asyncio>=3.4.3",
     "asyncstdlib>=3.13.0",
     "litellm>=1.60.0",
@@ -138,6 +139,10 @@ testpaths = [
 asyncio_mode = "auto"
 asyncio_default_fixture_loop_scope = "function"
 
+[[tool.uv.index]]
+url = "https://mirrors.bfsu.edu.cn/pypi/web/simple"
+default = true
+
 [[tool.uv.index]]
 url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple"
 
@@ -147,7 +152,3 @@ url = "https://mirrors.huaweicloud.com/repository/pypi/simple"
 [[tool.uv.index]]
 url = "https://mirrors.aliyun.com/pypi/simple/"
 
-[[tool.uv.index]]
-url = "https://mirrors.bfsu.edu.cn/pypi/web/simple"
-default = true
-

{fabricatio-0.2.4.dev3 → fabricatio-0.2.5.dev1}/python/fabricatio/__init__.py

@@ -2,10 +2,12 @@
 
 from importlib.util import find_spec
 
+from fabricatio._rust import BibManager
 from fabricatio._rust_instances import template_manager
-from fabricatio.actions.article import ExtractArticleEssence
+from fabricatio.actions.article import ExtractArticleEssence, GenerateArticleProposal, GenerateOutline
+from fabricatio.actions.output import DumpFinalizedOutput
 from fabricatio.core import env
-from fabricatio.fs import magika
+from fabricatio.fs import magika, safe_json_read, safe_text_read
 from fabricatio.journal import logger
 from fabricatio.models.action import Action, WorkFlow
 from fabricatio.models.events import Event
@@ -15,15 +17,20 @@ from fabricatio.models.task import Task
 from fabricatio.models.tool import ToolBox
 from fabricatio.models.utils import Message, Messages
 from fabricatio.parser import Capture, CodeBlockCapture, JsonCapture, PythonCapture
-from fabricatio.toolboxes import arithmetic_toolbox, basic_toolboxes, fs_toolbox, task_toolbox
+from fabricatio.toolboxes import arithmetic_toolbox, basic_toolboxes, fs_toolbox
+from fabricatio.workflows.articles import WriteOutlineWorkFlow
 
 __all__ = [
     "Action",
     "ArticleEssence",
+    "BibManager",
     "Capture",
     "CodeBlockCapture",
+    "DumpFinalizedOutput",
     "Event",
     "ExtractArticleEssence",
+    "GenerateArticleProposal",
+    "GenerateOutline",
     "JsonCapture",
     "Message",
     "Messages",
@@ -32,13 +39,15 @@ __all__ = [
     "Task",
     "ToolBox",
     "WorkFlow",
+    "WriteOutlineWorkFlow",
     "arithmetic_toolbox",
     "basic_toolboxes",
     "env",
     "fs_toolbox",
     "logger",
     "magika",
-    "task_toolbox",
+    "safe_json_read",
+    "safe_text_read",
     "template_manager",
 ]
 
@@ -46,6 +55,6 @@ __all__ = [
 if find_spec("pymilvus"):
     from fabricatio.actions.rag import InjectToDB
     from fabricatio.capabilities.rag import RAG
-    from fabricatio.workflows.articles import StoreArticle
+    from fabricatio.workflows.rag import StoreArticle
 
     __all__ += ["RAG", "InjectToDB", "StoreArticle"]

{fabricatio-0.2.4.dev3 → fabricatio-0.2.5.dev1}/python/fabricatio/_rust.pyi

@@ -51,3 +51,11 @@ def blake3_hash(content: bytes) -> str:
     Returns:
         str: The BLAKE3 hash of the data.
     """
+
+class BibManager:
+    """BibManager class for managing BibTeX files."""
+    def __init__(self, path: str) -> None:
+        """Initialize the BibManager with the given path."""
+
+    def get_cite_key(self, title: str) -> str:
+        """Get the cite key for the given title."""

fabricatio-0.2.5.dev1/python/fabricatio/actions/article.py

@@ -0,0 +1,81 @@
+"""Actions for transmitting tasks to targets."""
+
+from os import PathLike
+from pathlib import Path
+from typing import Callable, List
+
+from fabricatio.fs import safe_text_read
+from fabricatio.journal import logger
+from fabricatio.models.action import Action
+from fabricatio.models.extra import ArticleEssence, ArticleOutline, ArticleProposal
+from fabricatio.models.task import Task
+
+
+class ExtractArticleEssence(Action):
+    """Extract the essence of article(s) in text format from the paths specified in the task dependencies.
+
+    Notes:
+        This action is designed to extract vital information from articles with Markdown format, which is pure text, and
+        which is converted from pdf files using `magic-pdf` from the `MinerU` project, see https://github.com/opendatalab/MinerU
+    """
+
+    output_key: str = "article_essence"
+    """The key of the output data."""
+
+    async def _execute[P: PathLike | str](
+        self,
+        task_input: Task,
+        reader: Callable[[P], str] = lambda p: Path(p).read_text(encoding="utf-8"),
+        **_,
+    ) -> List[ArticleEssence]:
+        if not task_input.dependencies:
+            logger.info(err := "Task not approved, since no dependencies are provided.")
+            raise RuntimeError(err)
+
+        # trim the references
+        contents = ["References".join(c.split("References")[:-1]) for c in map(reader, task_input.dependencies)]
+        return await self.propose(
+            ArticleEssence,
+            contents,
+            system_message=f"# your personal briefing: \n{self.briefing}",
+        )
+
+
+class GenerateArticleProposal(Action):
+    """Generate an outline for the article based on the extracted essence."""
+
+    output_key: str = "article_proposal"
+    """The key of the output data."""
+
+    async def _execute(
+        self,
+        task_input: Task,
+        **_,
+    ) -> ArticleProposal:
+        input_path = await self.awhich_pathstr(
+            f"{task_input.briefing}\nExtract the path of file, which contains the article briefing that I need to read."
+        )
+
+        return await self.propose(
+            ArticleProposal,
+            safe_text_read(input_path),
+            system_message=f"# your personal briefing: \n{self.briefing}",
+        )
+
+
+class GenerateOutline(Action):
+    """Generate the article based on the outline."""
+
+    output_key: str = "article"
+    """The key of the output data."""
+
+    async def _execute(
+        self,
+        article_proposal: ArticleProposal,
+        **_,
+    ) -> ArticleOutline:
+        return await self.propose(
+            ArticleOutline,
+            article_proposal.display(),
+            system_message=f"# your personal briefing: \n{self.briefing}",
+        )

fabricatio-0.2.5.dev1/python/fabricatio/actions/output.py

@@ -0,0 +1,21 @@
+"""Dump the finalized output to a file."""
+
+from typing import Unpack
+
+from fabricatio.models.action import Action
+from fabricatio.models.generic import FinalizedDumpAble
+from fabricatio.models.task import Task
+
+
+class DumpFinalizedOutput(Action):
+    """Dump the finalized output to a file."""
+
+    output_key: str = "dump_path"
+
+    async def _execute(self, task_input: Task, to_dump: FinalizedDumpAble, **cxt: Unpack) -> str:
+        dump_path = await self.awhich_pathstr(
+            f"{task_input.briefing}\n\nExtract a single path of the file, to which I will dump the data."
+        )
+
+        to_dump.finalized_dump_to(dump_path)
+        return dump_path

{fabricatio-0.2.4.dev3 → fabricatio-0.2.5.dev1}/python/fabricatio/capabilities/rag.py

@@ -242,7 +242,6 @@ class RAG(EmbeddingUsage):
     async def aretrieve(
         self,
         query: List[str] | str,
-        collection_name: Optional[str] = None,
         final_limit: int = 20,
         **kwargs: Unpack[FetchKwargs],
     ) -> List[str]:
@@ -250,7 +249,6 @@ class RAG(EmbeddingUsage):
 
         Args:
             query (List[str] | str): The query to be used for retrieval.
-            collection_name (Optional[str]): The name of the collection. If not provided, the currently viewed collection is used.
             final_limit (int): The final limit on the number of results to return.
             **kwargs (Unpack[FetchKwargs]): Additional keyword arguments for retrieval.
 
@@ -263,15 +261,14 @@ class RAG(EmbeddingUsage):
             await self.afetch_document(
                 vecs=(await self.vectorize(query)),
                 desired_fields="text",
-                collection_name=collection_name,
                 **kwargs,
             )
         )[:final_limit]
 
     async def aask_retrieved(
         self,
-        question: str | List[str],
-        query: List[str] | str,
+        question: str,
+        query: Optional[List[str] | str] = None,
         collection_name: Optional[str] = None,
         extra_system_message: str = "",
         result_per_query: int = 10,
@@ -285,7 +282,7 @@ class RAG(EmbeddingUsage):
         specified question using the retrieved documents as context.
 
         Args:
-            question (str | List[str]): The question or list of questions to be asked.
+            question (str): The question to be asked.
             query (List[str] | str): The query or list of queries used for document retrieval.
             collection_name (Optional[str]): The name of the collection to retrieve documents from.
                                               If not provided, the currently viewed collection is used.
@@ -299,9 +296,9 @@ class RAG(EmbeddingUsage):
             str: A string response generated after asking with the context of retrieved documents.
         """
         docs = await self.aretrieve(
-            query,
-            collection_name,
+            query or question,
             final_limit,
+            collection_name=collection_name,
             result_per_query=result_per_query,
             similarity_threshold=similarity_threshold,
         )
@@ -332,3 +329,38 @@ class RAG(EmbeddingUsage):
             ),
             **kwargs,
         )
+
+    async def aask_refined(
+        self,
+        question: str,
+        collection_name: Optional[str] = None,
+        extra_system_message: str = "",
+        result_per_query: int = 10,
+        final_limit: int = 20,
+        similarity_threshold: float = 0.37,
+        **kwargs: Unpack[LLMKwargs],
+    ) -> str:
+        """Asks a question using a refined query based on the provided question.
+
+        Args:
+            question (str): The question to be asked.
+            collection_name (Optional[str]): The name of the collection to retrieve documents from.
+            extra_system_message (str): An additional system message to be included in the prompt.
+            result_per_query (int): The number of results to return per query. Default is 10.
+            final_limit (int): The maximum number of retrieved documents to consider. Default is 20.
+            similarity_threshold (float): The threshold for similarity, only results above this threshold will be returned.
+            **kwargs (Unpack[LLMKwargs]): Additional keyword arguments passed to the underlying `aask` method.
+
+        Returns:
+            str: A string response generated after asking with the refined question.
+        """
+        return await self.aask_retrieved(
+            question,
+            await self.arefined_query(question, **kwargs),
+            collection_name=collection_name,
+            extra_system_message=extra_system_message,
+            result_per_query=result_per_query,
+            final_limit=final_limit,
+            similarity_threshold=similarity_threshold,
+            **kwargs,
+        )

{fabricatio-0.2.4.dev3 → fabricatio-0.2.5.dev1}/python/fabricatio/capabilities/rating.py

@@ -4,7 +4,7 @@ from asyncio import gather
 from itertools import permutations
 from typing import Dict, List, Set, Tuple, Union, Unpack, overload
 
-import orjson
+from cache import AsyncLRU
 from fabricatio._rust_instances import template_manager
 from fabricatio.config import configs
 from fabricatio.journal import logger
@@ -40,10 +40,8 @@ class GiveRating(WithBriefing, LLMUsage):
 
         def _validator(response: str) -> Dict[str, float] | None:
             if (
-                (json_data := JsonCapture.convert_with(response, orjson.loads)) is not None
-                and isinstance(json_data, dict)
+                (json_data := JsonCapture.validate_with(response, dict, str))
                 and json_data.keys() == rating_manual.keys()
-                and all(isinstance(v, float) for v in json_data.values())
                 and all(score_range[0] <= v <= score_range[1] for v in json_data.values())
             ):
                 return json_data
@@ -115,6 +113,7 @@ class GiveRating(WithBriefing, LLMUsage):
             return await gather(*[self.rate_fine_grind(item, manual, score_range, **kwargs) for item in to_rate])
         raise ValueError("to_rate must be a string or a list of strings")
 
+    @AsyncLRU(maxsize=32)
     async def draft_rating_manual(
         self, topic: str, criteria: Set[str], **kwargs: Unpack[ValidateKwargs]
     ) -> Dict[str, str]:
@@ -169,16 +168,6 @@ class GiveRating(WithBriefing, LLMUsage):
         Returns:
             Set[str]: A set of rating dimensions.
         """
-
-        def _validator(response: str) -> Set[str] | None:
-            if (
-                json_data := JsonCapture.validate_with(
-                    response, target_type=list, elements_type=str, length=criteria_count
-                )
-            ) is not None:
-                return set(json_data)
-            return None
-
         return await self.aask_validate(
             question=(
                 template_manager.render_template(
@@ -189,7 +178,9 @@ class GiveRating(WithBriefing, LLMUsage):
                     },
                 )
             ),
-            validator=_validator,
+            validator=lambda resp: set(out)
+            if (out := JsonCapture.validate_with(resp, list, str, criteria_count))
+            else out,
             system_message=f"# your personal briefing: \n{self.briefing}",
             **kwargs,
         )