fabricatio 0.2.7.dev5__cp312-cp312-manylinux_2_34_x86_64.whl → 0.2.8__cp312-cp312-manylinux_2_34_x86_64.whl

fabricatio/__init__.py

@@ -1,10 +1,6 @@
 """Fabricatio is a Python library for building llm app using event-based agent structure."""
 
-from importlib.util import find_spec
-
-from fabricatio import actions, toolboxes, workflows
-from fabricatio._rust import BibManager
-from fabricatio._rust_instances import TEMPLATE_MANAGER
+from fabricatio import actions, capabilities, toolboxes, workflows
 from fabricatio.core import env
 from fabricatio.journal import logger
 from fabricatio.models import extra
@@ -14,6 +10,8 @@ from fabricatio.models.role import Role
 from fabricatio.models.task import Task
 from fabricatio.models.tool import ToolBox
 from fabricatio.parser import Capture, GenericCapture, JsonCapture, PythonCapture
+from fabricatio.rust import BibManager
+from fabricatio.rust_instances import TEMPLATE_MANAGER
 
 __all__ = [
     "TEMPLATE_MANAGER",
@@ -29,15 +27,10 @@ __all__ = [
     "ToolBox",
     "WorkFlow",
     "actions",
+    "capabilities",
     "env",
     "extra",
     "logger",
     "toolboxes",
     "workflows",
 ]
-
-
-if find_spec("pymilvus"):
-    from fabricatio.capabilities.rag import RAG
-
-    __all__ += ["RAG"]

fabricatio/actions/article.py

@@ -1,20 +1,28 @@
 """Actions for transmitting tasks to targets."""
 
+from asyncio import gather
 from pathlib import Path
 from typing import Any, Callable, List, Optional
 
+from fabricatio.capabilities.censor import Censor
+from fabricatio.capabilities.correct import Correct
+from fabricatio.capabilities.propose import Propose
 from fabricatio.fs import safe_text_read
 from fabricatio.journal import logger
 from fabricatio.models.action import Action
+from fabricatio.models.extra.article_base import ArticleRefSequencePatch
 from fabricatio.models.extra.article_essence import ArticleEssence
 from fabricatio.models.extra.article_main import Article
 from fabricatio.models.extra.article_outline import ArticleOutline
 from fabricatio.models.extra.article_proposal import ArticleProposal
+from fabricatio.models.extra.rule import RuleSet
 from fabricatio.models.task import Task
-from fabricatio.models.utils import ok
+from fabricatio.rust import BibManager
+from fabricatio.utils import ok
+from more_itertools import filter_map
 
 
-class ExtractArticleEssence(Action):
+class ExtractArticleEssence(Action, Propose):
     """Extract the essence of article(s) in text format from the paths specified in the task dependencies.
 
     Notes:
@@ -28,23 +36,65 @@ class ExtractArticleEssence(Action):
     async def _execute(
         self,
         task_input: Task,
-        reader: Callable[[str], str] = lambda p: Path(p).read_text(encoding="utf-8"),
+        reader: Callable[[str], Optional[str]] = lambda p: Path(p).read_text(encoding="utf-8"),
         **_,
-    ) -> Optional[List[ArticleEssence]]:
+    ) -> List[ArticleEssence]:
         if not task_input.dependencies:
             logger.info(err := "Task not approved, since no dependencies are provided.")
             raise RuntimeError(err)
-
+        logger.info(f"Extracting article essence from {len(task_input.dependencies)} files.")
         # trim the references
-        contents = ["References".join(c.split("References")[:-1]) for c in map(reader, task_input.dependencies)]
-        return await self.propose(
-            ArticleEssence,
-            contents,
-            system_message=f"# your personal briefing: \n{self.briefing}",
-        )
+        contents = list(filter_map(reader, task_input.dependencies))
+        logger.info(f"Read {len(task_input.dependencies)} to get {len(contents)} contents.")
 
+        out = []
+
+        for ess in await self.propose(
+            ArticleEssence,
+            [
+                f"{c}\n\n\nBased the provided academic article above, you need to extract the essence from it."
+                for c in contents
+            ],
+        ):
+            if ess is None:
+                logger.warning("Could not extract article essence")
+            else:
+                out.append(ess)
+        logger.info(f"Extracted {len(out)} article essence from {len(task_input.dependencies)} files.")
+        return out
+
+
+class FixArticleEssence(Action):
+    """Fix the article essence based on the bibtex key."""
+
+    output_key: str = "fixed_article_essence"
+    """The key of the output data."""
 
-class GenerateArticleProposal(Action):
+    async def _execute(
+        self,
+        bib_mgr: BibManager,
+        article_essence: List[ArticleEssence],
+        **_,
+    ) -> List[ArticleEssence]:
+        out = []
+        count = 0
+        for a in article_essence:
+            if key := (bib_mgr.get_cite_key(a.title) or bib_mgr.get_cite_key_fuzzy(a.title)):
+                a.title = bib_mgr.get_title_by_key(key) or a.title
+                a.authors = bib_mgr.get_author_by_key(key) or a.authors
+                a.publication_year = bib_mgr.get_year_by_key(key) or a.publication_year
+                a.bibtex_cite_key = key
+                logger.info(f"Updated {a.title} with {key}")
+                out.append(a)
+            else:
+                logger.warning(f"No key found for {a.title}")
+                count += 1
+        if count:
+            logger.warning(f"{count} articles have no key")
+        return out
+
+
+class GenerateArticleProposal(Action, Propose):
     """Generate an outline for the article based on the extracted essence."""
 
     output_key: str = "article_proposal"
@@ -55,13 +105,14 @@ class GenerateArticleProposal(Action):
         task_input: Optional[Task] = None,
         article_briefing: Optional[str] = None,
         article_briefing_path: Optional[str] = None,
+        langauge: Optional[str] = None,
         **_,
     ) -> Optional[ArticleProposal]:
         if article_briefing is None and article_briefing_path is None and task_input is None:
             logger.error("Task not approved, since all inputs are None.")
             return None
 
-        return ok(
+        proposal = ok(
             await self.propose(
                 ArticleProposal,
                 briefing := (
@@ -76,16 +127,19 @@ class GenerateArticleProposal(Action):
                         )
                     )
                 ),
-                **self.prepend_sys_msg(),
             ),
-            "Could not generate the proposal."
+            "Could not generate the proposal.",
         ).update_ref(briefing)
+        if langauge:
+            proposal.language = langauge
 
+        return proposal
 
-class GenerateOutline(Action):
-    """Generate the article based on the outline."""
 
-    output_key: str = "article_outline"
+class GenerateInitialOutline(Action, Propose):
+    """Generate the initial article outline based on the article proposal."""
+
+    output_key: str = "initial_article_outline"
     """The key of the output data."""
 
     async def _execute(
@@ -93,118 +147,191 @@ class GenerateOutline(Action):
         article_proposal: ArticleProposal,
         **_,
     ) -> Optional[ArticleOutline]:
-        out = ok(
+        return ok(
             await self.propose(
                 ArticleOutline,
                 article_proposal.as_prompt(),
-                **self.prepend_sys_msg(),
             ),
-            "Could not generate the outline.",
-        )
+            "Could not generate the initial outline.",
+        ).update_ref(article_proposal)
 
-        manual = ok(await self.draft_rating_manual(
-            topic=(
-                topic
-                := "Fix the internal referring error, make sure there is no more `ArticleRef` pointing to a non-existing article component."
-            ),
-        ),"Could not generate the rating manual.")
 
-        while pack := out.find_illegal():
-            component, err = ok(pack)
-            logger.warning(f"Found error in the outline: \n{err}")
-            corrected = ok(await self.correct_obj(
-                component,
-                reference=f"# Original Article Outline\n{out.display()}\n# Error Need to be fixed\n{err}",
-                topic=topic,
-                rating_manual=manual,
-                supervisor_check=False,
-            ),"Could not correct the component.")
-            component.update_from(corrected)
-        return out.update_ref(article_proposal)
+class FixIntrospectedErrors(Action, Censor):
+    """Fix introspected errors in the article outline."""
 
+    output_key: str = "introspected_errors_fixed_outline"
+    """The key of the output data."""
 
-class CorrectProposal(Action):
-    """Correct the proposal of the article."""
+    ruleset: Optional[RuleSet] = None
+    """The ruleset to use to fix the introspected errors."""
 
-    output_key: str = "corrected_proposal"
+    async def _execute(
+        self,
+        article_outline: ArticleOutline,
+        ruleset: Optional[RuleSet] = None,
+        **_,
+    ) -> Optional[ArticleOutline]:
+        while pack := article_outline.find_introspected():
+            component, err = ok(pack)
+            logger.warning(f"Found introspected error: {err}")
+            corrected = ok(
+                await self.censor_obj(
+                    component,
+                    ruleset=ok(ruleset or self.ruleset, "No ruleset provided"),
+                    reference=f"# Original Article Outline\n{article_outline.display()}\n# Some Basic errors found from `{component.title}` that need to be fixed\n{err}",
+                ),
+                "Could not correct the component.",
+            )
+            component.update_from(corrected)
 
-    async def _execute(self, article_proposal: ArticleProposal, **_) -> Any:
-        return (await self.censor_obj(article_proposal, reference=article_proposal.referenced)).update_ref(
-            article_proposal
-        )
+        return article_outline
 
 
-class CorrectOutline(Action):
-    """Correct the outline of the article."""
+class FixIllegalReferences(Action, Censor):
+    """Fix illegal references in the article outline."""
 
-    output_key: str = "corrected_outline"
+    output_key: str = "illegal_references_fixed_outline"
     """The key of the output data."""
 
+    ruleset: Optional[RuleSet] = None
+    """Ruleset to use to fix the illegal references."""
+
     async def _execute(
         self,
         article_outline: ArticleOutline,
+        ruleset: Optional[RuleSet] = None,
         **_,
+    ) -> Optional[ArticleOutline]:
+        while pack := article_outline.find_illegal_ref(gather_identical=True):
+            refs, err = ok(pack)
+            logger.warning(f"Found illegal referring error: {err}")
+            corrected_ref = ok(
+                await self.censor_obj(
+                    refs[0],  # pyright: ignore [reportIndexIssue]
+                    ruleset=ok(ruleset or self.ruleset, "No ruleset provided"),
+                    reference=f"# Original Article Outline\n{article_outline.display()}\n# Some Basic errors found that need to be fixed\n{err}",
+                )
+            )
+            for ref in refs:
+                ref.update_from(corrected_ref)  # pyright: ignore [reportAttributeAccessIssue]
+
+        return article_outline.update_ref(article_outline)
+
+
+class TweakOutlineForwardRef(Action, Censor):
+    """Tweak the forward references in the article outline.
+
+    Ensures that the conclusions of the current chapter effectively support the analysis of subsequent chapters.
+    """
+
+    output_key: str = "article_outline_fw_ref_checked"
+    ruleset: Optional[RuleSet] = None
+    """Ruleset to use to fix the illegal references."""
+
+    async def _execute(
+        self, article_outline: ArticleOutline, ruleset: Optional[RuleSet] = None, **cxt
     ) -> ArticleOutline:
-        return (await self.censor_obj(article_outline, reference=article_outline.referenced.as_prompt())).update_ref(
-            article_outline
+        return await self._inner(
+            article_outline,
+            ruleset=ok(ruleset or self.ruleset, "No ruleset provided"),
+            field_name="support_to",
         )
 
+    async def _inner(self, article_outline: ArticleOutline, ruleset: RuleSet, field_name: str) -> ArticleOutline:
+        for a in article_outline.iter_dfs():
+            if judge := await self.evidently_judge(
+                f"{article_outline.as_prompt()}\n\n{a.display()}\n"
+                f"Does the `{a.__class__.__name__}`'s `{field_name}` field need to be extended or tweaked?"
+            ):
+                patch = ArticleRefSequencePatch.default()
+                patch.tweaked = getattr(a, field_name)
+
+                await self.censor_obj_inplace(
+                    patch,
+                    ruleset=ruleset,
+                    reference=f"{article_outline.as_prompt()}\n"
+                    f"The Article component titled `{a.title}` whose `{field_name}` field needs to be extended or tweaked.\n"
+                    f"# Judgement\n{judge.display()}",
+                )
+        return article_outline
+
+
+class TweakOutlineBackwardRef(TweakOutlineForwardRef):
+    """Tweak the backward references in the article outline.
+
+    Ensures that the prerequisites of the current chapter are correctly referenced in the `depend_on` field.
+    """
+
+    output_key: str = "article_outline_bw_ref_checked"
+    ruleset: Optional[RuleSet] = None
 
-class GenerateArticle(Action):
+    async def _execute(
+        self, article_outline: ArticleOutline, ruleset: Optional[RuleSet] = None, **cxt
+    ) -> ArticleOutline:
+        return await self._inner(
+            article_outline,
+            ruleset=ok(ruleset or self.ruleset, "No ruleset provided"),
+            field_name="depend_on",
+        )
+
+
+class GenerateArticle(Action, Censor):
     """Generate the article based on the outline."""
 
     output_key: str = "article"
     """The key of the output data."""
+    ruleset: Optional[RuleSet] = None
 
     async def _execute(
         self,
         article_outline: ArticleOutline,
+        ruleset: Optional[RuleSet] = None,
         **_,
     ) -> Optional[Article]:
-        article: Article = Article.from_outline(article_outline).update_ref(article_outline)
+        article: Article = Article.from_outline(ok(article_outline, "Article outline not specified.")).update_ref(
+            article_outline
+        )
 
-        writing_manual = ok(await self.draft_rating_manual(
-            topic=(
-                topic_1
-                := "improve the content of the subsection to fit the outline. SHALL never add or remove any section or subsection, you can only add or delete paragraphs within the subsection."
-            ),
-        ))
-        err_resolve_manual = ok(await self.draft_rating_manual(
-            topic=(topic_2 := "this article component has violated the constrain, please correct it.")
-        ))
-        for c, deps in article.iter_dfs_with_deps(chapter=False):
-            logger.info(f"Updating the article component: \n{c.display()}")
-
-            out = ok(
-                await self.correct_obj(
-                    c,
-                    reference=(
-                        ref := f"{article_outline.referenced.as_prompt()}\n" + "\n".join(d.display() for d in deps)
-                    ),
-                    topic=topic_1,
-                    rating_manual=writing_manual,
-                    supervisor_check=False,
-                ),
-                "Could not correct the article component.",
-            )
-            while err := c.resolve_update_conflict(out):
-                logger.warning(f"Found error in the article component: \n{err}")
-                out = ok(
-                    await self.correct_obj(
-                        out,
-                        reference=f"{ref}\n\n# Violated Error\n{err}",
-                        topic=topic_2,
-                        rating_manual=err_resolve_manual,
-                        supervisor_check=False,
-                    ),
-                    "Could not correct the article component.",
+        await gather(
+            *[
+                self.censor_obj_inplace(
+                    subsec,
+                    ruleset=ok(ruleset or self.ruleset, "No ruleset provided"),
+                    reference=f"# Original Article Outline\n{article_outline.display()}\n# Error Need to be fixed\n{err}",
                 )
+                for _, __, subsec in article.iter_subsections()
+                if (err := subsec.introspect())
+            ],
+            return_exceptions=True,
+        )
 
-            c.update_from(out)
         return article
 
 
-class CorrectArticle(Action):
+class CorrectProposal(Action, Censor):
+    """Correct the proposal of the article."""
+
+    output_key: str = "corrected_proposal"
+
+    async def _execute(self, article_proposal: ArticleProposal, **_) -> Any:
+        raise  NotImplementedError("Not implemented.")
+
+
+class CorrectOutline(Action, Correct):
+    """Correct the outline of the article."""
+
+    output_key: str = "corrected_outline"
+    """The key of the output data."""
+
+    async def _execute(
+        self,
+        article_outline: ArticleOutline,
+        **_,
+    ) -> ArticleOutline:
+        raise NotImplementedError("Not implemented.")
+
+
+class CorrectArticle(Action, Correct):
     """Correct the article based on the outline."""
 
     output_key: str = "corrected_article"
@@ -216,4 +343,4 @@ class CorrectArticle(Action):
         article_outline: ArticleOutline,
         **_,
     ) -> Article:
-        return await self.censor_obj(article, reference=article_outline.referenced.as_prompt())
+        raise NotImplementedError("Not implemented.")

fabricatio/actions/article_rag.py

@@ -1,35 +1,100 @@
 """A module for writing articles using RAG (Retrieval-Augmented Generation) capabilities."""
 
+from asyncio import gather
 from typing import Optional
 
+from fabricatio.capabilities.censor import Censor
 from fabricatio.capabilities.rag import RAG
-from fabricatio.journal import logger
 from fabricatio.models.action import Action
-from fabricatio.models.extra.article_main import Article
-from fabricatio.models.extra.article_outline import ArticleOutline
+from fabricatio.models.extra.article_main import Article, ArticleParagraphSequencePatch, ArticleSubsection
+from fabricatio.models.extra.rule import RuleSet
+from fabricatio.utils import ok
 
 
-class GenerateArticleRAG(Action, RAG):
-    """Write an article based on the provided outline."""
+class TweakArticleRAG(Action, RAG, Censor):
+    """Write an article based on the provided outline.
 
-    output_key: str = "article"
+    This class inherits from `Action`, `RAG`, and `Censor` to provide capabilities for writing and refining articles
+    using Retrieval-Augmented Generation (RAG) techniques. It processes an article outline, enhances subsections by
+    searching for related references, and applies censoring rules to ensure compliance with the provided ruleset.
 
-    async def _execute(self, article_outline: ArticleOutline, **cxt) -> Optional[Article]:
-        """Write an article based on the provided outline."""
-        logger.info(f"Writing an article based on the outline:\n{article_outline.title}")
-        refined_q = await self.arefined_query(article_outline.display())
-        return await self.propose(
-            Article,
-            article_outline.display(),
-            **self.prepend_sys_msg(f"{await self.aretrieve_compact(refined_q)}\n{self.briefing}"),
-        )
+    Attributes:
+        output_key (str): The key used to store the output of the action.
+        ruleset (Optional[RuleSet]): The ruleset to be used for censoring the article.
+    """
+
+    output_key: str = "rag_tweaked_article"
+    """The key used to store the output of the action."""
+
+    ruleset: Optional[RuleSet] = None
+    """The ruleset to be used for censoring the article."""
+
+    async def _execute(
+        self,
+        article: Article,
+        collection_name: str = "article_essence",
+        ruleset: Optional[RuleSet] = None,
+        parallel: bool = False,
+        **cxt,
+    ) -> Optional[Article]:
+        """Write an article based on the provided outline.
+
+        This method processes the article outline, either in parallel or sequentially, by enhancing each subsection
+        with relevant references and applying censoring rules.
 
+        Args:
+            article (Article): The article to be processed.
+            collection_name (str): The name of the collection to view for processing.
+            ruleset (Optional[RuleSet]): The ruleset to apply for censoring. If not provided, the class's ruleset is used.
+            parallel (bool): If True, process subsections in parallel. Otherwise, process them sequentially.
+            **cxt: Additional context parameters.
 
-class WriteArticleFineGrind(Action, RAG):
-    """Fine-grind an article based on the provided outline."""
+        Returns:
+            Optional[Article]: The processed article with enhanced subsections and applied censoring rules.
+        """
+        self.view(collection_name)
 
-    output_key: str = "article"
+        if parallel:
+            await gather(
+                *[
+                    self._inner(article, subsec, ok(ruleset or self.ruleset, "No ruleset provided!"))
+                    for _, __, subsec in article.iter_subsections()
+                ],
+                return_exceptions=True,
+            )
+        else:
+            for _, __, subsec in article.iter_subsections():
+                await self._inner(article, subsec, ok(ruleset or self.ruleset, "No ruleset provided!"))
+        return article
 
-    async def _execute(self, article_outline: ArticleOutline, **cxt) -> Optional[Article]:
-        """Fine-grind an article based on the provided outline."""
-        logger.info(f"Fine-grinding an article based on the outline:\n{article_outline.title}")
+    async def _inner(self, article: Article, subsec: ArticleSubsection, ruleset: RuleSet) -> None:
+        """Enhance a subsection of the article with references and apply censoring rules.
+
+        This method refines the query for the subsection, retrieves related references, and applies censoring rules
+        to the subsection's paragraphs.
+
+        Args:
+            article (Article): The article containing the subsection.
+            subsec (ArticleSubsection): The subsection to be enhanced.
+            ruleset (RuleSet): The ruleset to apply for censoring.
+
+        Returns:
+            None
+        """
+        refind_q = ok(
+            await self.arefined_query(
+                f"{article.referenced.as_prompt()}\n"
+                f"# Subsection requiring reference enhancement\n"
+                f"{subsec.display()}\n"
+                f"# Requirement\n"
+                f"Search related articles in the base to find reference candidates, "
+                f"prioritizing both original article language and English usage",
+            )
+        )
+        patch = ArticleParagraphSequencePatch.default()
+        patch.tweaked = subsec.paragraphs
+        await self.censor_obj_inplace(
+            patch,
+            ruleset=ruleset,
+            reference=await self.aretrieve_compact(refind_q, final_limit=30),
+        )

fabricatio/actions/output.py

@@ -1,18 +1,20 @@
 """Dump the finalized output to a file."""
 
 from pathlib import Path
-from typing import Optional
+from typing import Iterable, List, Optional, Type
 
+from fabricatio.journal import logger
 from fabricatio.models.action import Action
-from fabricatio.models.generic import FinalizedDumpAble
+from fabricatio.models.generic import FinalizedDumpAble, PersistentAble
 from fabricatio.models.task import Task
-from fabricatio.models.utils import ok
+from fabricatio.utils import ok
 
 
 class DumpFinalizedOutput(Action):
     """Dump the finalized output to a file."""
 
     output_key: str = "dump_path"
+    dump_path: Optional[str] = None
 
     async def _execute(
         self,
@@ -23,6 +25,7 @@ class DumpFinalizedOutput(Action):
     ) -> str:
         dump_path = Path(
             dump_path
+            or self.dump_path
             or ok(
                 await self.awhich_pathstr(
                     f"{ok(task_input, 'Neither `task_input` and `dump_path` is provided.').briefing}\n\nExtract a single path of the file, to which I will dump the data."
@@ -32,3 +35,68 @@ class DumpFinalizedOutput(Action):
         )
         ok(to_dump, "Could not dump the data since the path is not specified.").finalized_dump_to(dump_path)
         return dump_path.as_posix()
+
+
+class PersistentAll(Action):
+    """Persist all the data to a file."""
+
+    output_key: str = "persistent_count"
+    persist_dir: Optional[str] = None
+
+    async def _execute(
+        self,
+        task_input: Optional[Task] = None,
+        persist_dir: Optional[str | Path] = None,
+        **cxt,
+    ) -> int:
+        persist_dir = Path(
+            persist_dir
+            or self.persist_dir
+            or ok(
+                await self.awhich_pathstr(
+                    f"{ok(task_input, 'Neither `task_input` and `dump_path` is provided.').briefing}\n\nExtract a single path of the file, to which I will persist the data."
+                ),
+                "Can not find the path of file to persist the data.",
+            )
+        )
+
+        count = 0
+        if persist_dir.is_file():
+            logger.warning("Dump should be a directory, but it is a file. Skip dumping.")
+            return count
+
+        for k, v in cxt.items():
+            final_dir = persist_dir.joinpath(k)
+            if isinstance(v, PersistentAble):
+                final_dir.mkdir(parents=True, exist_ok=True)
+                v.persist(final_dir)
+                count += 1
+            if isinstance(v, Iterable) and any(
+                persistent_ables := (pers for pers in v if isinstance(pers, PersistentAble))
+            ):
+                final_dir.mkdir(parents=True, exist_ok=True)
+                for per in persistent_ables:
+                    per.persist(final_dir)
+                    count += 1
+        logger.info(f"Persisted {count} objects to {persist_dir}")
+        return count
+
+
+class RetrieveFromPersistent[T: PersistentAble](Action):
+    """Retrieve the object from the persistent file."""
+
+    output_key: str = "retrieved_obj"
+    """Retrieve the object from the persistent file."""
+    load_path: str
+    """The path of the persistent file or directory contains multiple file."""
+    retrieve_cls: Type[T]
+    """The class of the object to retrieve."""
+
+    async def _execute(self, /, **__) -> Optional[T | List[T]]:
+        logger.info(f"Retrieve `{self.retrieve_cls.__name__}` from persistent file: {self.load_path}")
+        if not (p := Path(self.load_path)).exists():
+            return None
+
+        if p.is_dir():
+            return [self.retrieve_cls.from_persistent(per) for per in p.glob("*")]
+        return self.retrieve_cls.from_persistent(self.load_path)