fabricatio 0.2.13.dev1__tar.gz → 0.2.13.dev3__tar.gz

{fabricatio-0.2.13.dev1 → fabricatio-0.2.13.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fabricatio
-Version: 0.2.13.dev1
+Version: 0.2.13.dev3
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: 3.12

{fabricatio-0.2.13.dev1 → fabricatio-0.2.13.dev3}/examples/write_article/write_article.py

@@ -26,10 +26,9 @@ from typer import Typer
 Role(
     name="Undergraduate Researcher",
     description="Write an outline for an article in typst format.",
-    llm_model="openai/qwen-plus",
-    llm_temperature=0.45,
+    llm_model="openai/qwen3-235b-a22b",
     # llm_api_endpoint=HttpUrl("https://dashscope.aliyuncs.com/compatible-mode/v1"),
-    llm_top_p=0.95,
+    llm_stream=True,
     llm_max_tokens=8191,
     llm_rpm=600,
     llm_tpm=900000,
@@ -49,7 +48,7 @@ Role(
                         threshold=0.58,
                         result_per_query=2,
                         extractor_model={"model": "openai/qwen-max"},
-                        query_model={"model": "openai/qwen-turbo", "temperature": 0.3, "top_p": 0.85},
+                        query_model={"model": "openai/qwen-turbo"},
                     )
                 ),
                 PersistentAll,
@@ -212,6 +211,7 @@ def suma(
     ),
     dump_path: Path = typer.Option(Path("out.typ"), "-d", "--dump-path", help="Path to dump the final output."),  # noqa: B008
 ) -> None:
+    """Write chap summary based on given article."""
     path = ok(
         asyncio.run(
             Task(name="write an article")

{fabricatio-0.2.13.dev1 → fabricatio-0.2.13.dev3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "fabricatio"
-version = "0.2.13-dev1"
+version = "0.2.13-dev3"
 description = "A LLM multi-agent framework."
 readme = "README.md"
 license = { file = "LICENSE" }

{fabricatio-0.2.13.dev1 → fabricatio-0.2.13.dev3}/python/fabricatio/actions/article_rag.py

@@ -20,18 +20,26 @@ from fabricatio.models.extra.article_main import Article, ArticleChapter, Articl
 from fabricatio.models.extra.article_outline import ArticleOutline
 from fabricatio.models.extra.rule import RuleSet
 from fabricatio.models.kwargs_types import ChooseKwargs, LLMKwargs
-from fabricatio.rust import convert_to_block_formula, convert_to_inline_formula
-from fabricatio.utils import ask_retain, ok
+from fabricatio.rust import (
+    convert_all_block_tex,
+    convert_all_inline_tex,
+    convert_to_block_formula,
+    convert_to_inline_formula,
+    fix_misplaced_labels,
+)
+from fabricatio.utils import ok
 
 TYPST_CITE_USAGE = (
-    "citation number is REQUIRED to cite any reference!,for example in Auther Pattern: 'Doe et al.[[1]], Jack et al.[[2]]' or in Sentence Suffix Sattern: 'Global requirement is incresing[[1]].'\n"
-    "Everything is build upon the typst language, which is similar to latex, \n"
+    "citation number is REQUIRED to cite any reference!'\n"
     "Legal citing syntax examples(seperated by |): [[1]]|[[1,2]]|[[1-3]]|[[12,13-15]]|[[1-3,5-7]]\n"
     "Illegal citing syntax examples(seperated by |): [[1],[2],[3]]|[[1],[1-2]]\n"
-    "Those reference mark shall not be omitted during the extraction\n"
+    "You SHALL not cite a single reference more than once!"
     "It's recommended to cite multiple references that supports your conclusion at a time.\n"
+)
+
+TYPST_MATH_USAGE = (
     "Wrap inline expression with '\\(' and '\\)',like '\\(>5m\\)' '\\(89%\\)', and wrap block equation with '\\[' and '\\]'.\n"
-    "In addition to that, you can add a label outside the block equation which can be used as a cross reference identifier, the label is a string wrapped in `<` and `>` like `<energy-release-rate-equation>`.Note that the label string should be a summarizing title for the equation being labeled.\n"
+    "In addition to that, you can add a label outside the block equation which can be used as a cross reference identifier, the label is a string wrapped in `<` and `>` like `<energy-release-rate-equation>`.Note that the label string should be a summarizing title for the equation being labeled and should never be written within the formula block.\n"
     "you can refer to that label by using the syntax with prefix of `@eqt:`, which indicate that this notation is citing a label from the equations. For example ' @eqt:energy-release-rate-equation ' DO remember that the notation shall have both suffixed and prefixed space char which enable the compiler to distinguish the notation from the plaintext."
     "Below is two usage example:\n"
     "```typst\n"
@@ -44,9 +52,10 @@ TYPST_CITE_USAGE = (
 )
 
 
-class WriteArticleContentRAG(Action, RAG, Extract):
+class WriteArticleContentRAG(Action, Extract, AdvancedRAG):
     """Write an article based on the provided outline."""
 
+    ctx_override: ClassVar[bool] = True
     search_increment_multiplier: float = 1.6
     """The increment multiplier of the search increment."""
     ref_limit: int = 35
@@ -55,15 +64,19 @@ class WriteArticleContentRAG(Action, RAG, Extract):
     """The threshold of relevance"""
     extractor_model: LLMKwargs
     """The model to use for extracting the content from the retrieved references."""
-    query_model: LLMKwargs
+    query_model: ChooseKwargs
     """The model to use for querying the database"""
     supervisor: bool = False
     """Whether to use supervisor mode"""
     result_per_query: PositiveInt = 4
     """The number of results to be returned per query."""
-    req: str = TYPST_CITE_USAGE
+    cite_req: str = TYPST_CITE_USAGE
     """The req of the write article content."""
 
+    math_req: str = TYPST_MATH_USAGE
+    """The req of the write article content."""
+    tei_endpoint: Optional[str] = None
+
     async def _execute(
         self,
         article_outline: ArticleOutline,
@@ -103,30 +116,25 @@ class WriteArticleContentRAG(Action, RAG, Extract):
         cm = CitationManager()
         await self.search_database(article, article_outline, chap, sec, subsec, cm)
 
-        raw = await self.write_raw(article, article_outline, chap, sec, subsec, cm)
-        r_print(raw)
+        raw_paras = await self.write_raw(article, article_outline, chap, sec, subsec, cm)
+        r_print(raw_paras)
 
         while not await confirm("Accept this version and continue?").ask_async():
             if inst := await text("Search for more refs for additional spec.").ask_async():
-                await self.search_database(
-                    article,
-                    article_outline,
-                    chap,
-                    sec,
-                    subsec,
-                    cm,
-                    supervisor=True,
-                    extra_instruction=inst,
-                )
+                await self.search_database(article, article_outline, chap, sec, subsec, cm, extra_instruction=inst)
 
             if instruction := await text("Enter the instructions to improve").ask_async():
-                raw = await self.write_raw(article, article_outline, chap, sec, subsec, cm, instruction)
-            if edt := await text("Edit", default=raw).ask_async():
-                raw = edt
+                raw_paras = await self.write_raw(article, article_outline, chap, sec, subsec, cm, instruction)
+            if edt := await text("Edit", default=raw_paras).ask_async():
+                raw_paras = edt
 
-            r_print(raw)
+            raw_paras = fix_misplaced_labels(raw_paras)
+            raw_paras = convert_all_inline_tex(raw_paras)
+            raw_paras = convert_all_block_tex(raw_paras)
 
-        return await self.extract_new_subsec(subsec, raw, cm)
+            r_print(raw_paras)
+
+        return await self.extract_new_subsec(subsec, raw_paras, cm)
 
     async def _inner(
         self,
@@ -144,6 +152,10 @@ class WriteArticleContentRAG(Action, RAG, Extract):
 
         raw_paras = "\n".join(p for p in raw_paras.splitlines() if p and not p.endswith("**") and not p.startswith("#"))
 
+        raw_paras = fix_misplaced_labels(raw_paras)
+        raw_paras = convert_all_inline_tex(raw_paras)
+        raw_paras = convert_all_block_tex(raw_paras)
+
         return await self.extract_new_subsec(subsec, raw_paras, cm)
 
     async def extract_new_subsec(
@@ -155,12 +167,13 @@ class WriteArticleContentRAG(Action, RAG, Extract):
                 ArticleSubsection,
                 raw_paras,
                 f"Above is the subsection titled `{subsec.title}`.\n"
-                f"I need you to extract the content to update my subsection obj provided below.\n{self.req}"
-                f"{subsec.display()}\n",
+                f"I need you to extract the content to construct a new `{ArticleSubsection.__class__.__name__}`,"
+                f"Do not attempt to change the original content, your job is ONLY content extraction",
                 **self.extractor_model,
             ),
             "Failed to propose new subsection.",
         )
+
         for p in new_subsec.paragraphs:
             p.content = cm.apply(p.content)
             p.description = cm.apply(p.description)
@@ -184,7 +197,7 @@ class WriteArticleContentRAG(Action, RAG, Extract):
             f"{article_outline.finalized_dump()}\n\nAbove is my article outline, I m writing graduate thesis titled `{article.title}`. "
             f"More specifically, i m witting the Chapter `{chap.title}` >> Section `{sec.title}` >> Subsection `{subsec.title}`.\n"
             f"Please help me write the paragraphs of the subsec mentioned above, which is `{subsec.title}`.\n"
-            f"{self.req}\n"
+            f"{self.cite_req}\n{self.math_req}\n"
             f"You SHALL use `{article.language}` as writing language.\n{extra_instruction}\n"
             f"Do not use numbered list to display the outcome, you should regard you are writing the main text of the thesis.\n"
             f"You should not copy others' works from the references directly on to my thesis, we can only harness the conclusion they have drawn.\n"
@@ -200,7 +213,6 @@ class WriteArticleContentRAG(Action, RAG, Extract):
         subsec: ArticleSubsection,
         cm: CitationManager,
         extra_instruction: str = "",
-        supervisor: bool = False,
     ) -> None:
         """Search database for related references."""
         search_req = (
@@ -208,51 +220,20 @@ class WriteArticleContentRAG(Action, RAG, Extract):
             f"More specifically, i m witting the Chapter `{chap.title}` >> Section `{sec.title}` >> Subsection `{subsec.title}`.\n"
             f"I need to search related references to build up the content of the subsec mentioned above, which is `{subsec.title}`.\n"
             f"provide 10~16 queries as possible, to get best result!\n"
-            f"You should provide both English version and chinese version of the refined queries!\n{extra_instruction}\n"
-        )
-
-        ref_q = ok(
-            await self.arefined_query(
-                search_req,
-                **self.query_model,
-            ),
-            "Failed to refine query.",
+            f"You should provide both English version and chinese version of the refined queries!\n{extra_instruction}"
         )
 
-        if supervisor:
-            ref_q = await ask_retain(ref_q)
-        ret = await self.aretrieve(
-            ref_q,
-            ArticleChunk,
-            max_accepted=self.ref_limit,
+        await self.clued_search(
+            search_req,
+            cm,
+            refinery_kwargs=self.query_model,
+            expand_multiplier=self.search_increment_multiplier,
+            base_accepted=self.ref_limit,
             result_per_query=self.result_per_query,
             similarity_threshold=self.threshold,
+            tei_endpoint=self.tei_endpoint,
         )
 
-        cm.add_chunks(ok(ret))
-        ref_q = await self.arefined_query(
-            f"{cm.as_prompt()}\n\nAbove is the retrieved references in the first RAG, now we need to perform the second RAG.\n\n{search_req}",
-            **self.query_model,
-        )
-
-        if ref_q is None:
-            logger.warning("Second refine query is None, skipping.")
-            return
-        if supervisor:
-            ref_q = await ask_retain(ref_q)
-
-        ret = await self.aretrieve(
-            ref_q,
-            ArticleChunk,
-            max_accepted=int(self.ref_limit * self.search_increment_multiplier),
-            result_per_query=int(self.result_per_query * self.search_increment_multiplier),
-            similarity_threshold=self.threshold,
-        )
-        if ret is None:
-            logger.warning("Second retrieve is None, skipping.")
-            return
-        cm.add_chunks(ret)
-
 
 class ArticleConsultRAG(Action, AdvancedRAG):
     """Write an article based on the provided outline."""
@@ -261,9 +242,9 @@ class ArticleConsultRAG(Action, AdvancedRAG):
     output_key: str = "consult_count"
     search_increment_multiplier: float = 1.6
     """The multiplier to increase the limit of references to retrieve per query."""
-    ref_limit: int = 20
+    ref_limit: int = 26
     """The final limit of references."""
-    ref_per_q: int = 3
+    ref_per_q: int = 13
     """The limit of references to retrieve per query."""
     similarity_threshold: float = 0.62
     """The similarity threshold of references to retrieve."""

{fabricatio-0.2.13.dev1 → fabricatio-0.2.13.dev3}/python/fabricatio/capabilities/advanced_rag.py

@@ -7,6 +7,7 @@ from fabricatio.journal import logger
 from fabricatio.models.adv_kwargs_types import FetchKwargs
 from fabricatio.models.extra.aricle_rag import ArticleChunk, CitationManager
 from fabricatio.models.kwargs_types import ChooseKwargs
+from fabricatio.utils import fallback_kwargs
 
 
 class AdvancedRAG(RAG):
@@ -40,10 +41,13 @@ class AdvancedRAG(RAG):
                 f"\n\n{requirement}",
                 **refinery_kwargs,
             )
+
             if ref_q is None:
                 logger.error(f"At round [{i}/{max_round}] search, failed to refine the query, exit.")
                 return cm
-            refs = await self.aretrieve(ref_q, ArticleChunk, base_accepted, **kwargs)
+            refs = await self.aretrieve(
+                ref_q, ArticleChunk, base_accepted, **fallback_kwargs(kwargs, filter_expr=cm.as_milvus_filter_expr())
+            )
 
             if (max_capacity := max_capacity - len(refs)) < 0:
                 cm.add_chunks(refs[0:max_capacity])

{fabricatio-0.2.13.dev1 → fabricatio-0.2.13.dev3}/python/fabricatio/capabilities/rag.py

@@ -150,6 +150,7 @@ class RAG(EmbeddingUsage):
         result_per_query: int = 10,
         tei_endpoint: Optional[str] = None,
         reranker_threshold: float = 0.7,
+        filter_expr: str = "",
     ) -> List[D]:
         """Asynchronously fetches documents from a Milvus database based on input vectors.
 
@@ -162,6 +163,7 @@ class RAG(EmbeddingUsage):
            result_per_query (int): The maximum number of results to return per query. Defaults to 10.
            tei_endpoint (str): the endpoint of the TEI api.
            reranker_threshold (float): The threshold used to filtered low relativity document.
+            filter_expr (str): filter_expression parsed into pymilvus search.
 
         Returns:
            List[D]: A list of document objects created from the fetched data.
@@ -172,6 +174,7 @@ class RAG(EmbeddingUsage):
             await self.vectorize(query),
             search_params={"radius": similarity_threshold},
             output_fields=list(document_model.model_fields),
+            filter=filter_expr,
             limit=result_per_query,
         )
         if tei_endpoint is not None:
@@ -184,8 +187,11 @@ class RAG(EmbeddingUsage):
 
             for q, g in zip(query, search_results, strict=True):
                 models = document_model.from_sequence([res["entity"] for res in g if res["id"] not in retrieved_id])
+                logger.debug(f"Retrived {len(g)} raw document, filtered out {len(models)}.")
                 retrieved_id.update(res["id"] for res in g)
-                rank_scores = await reranker.arerank(q, [m.prepare_vectorization() for m in models])
+                if not models:
+                    continue
+                rank_scores = await reranker.arerank(q, [m.prepare_vectorization() for m in models], truncate=True)
                 raw_result.extend(
                     (models[s["index"]], s["score"]) for s in rank_scores if s["score"] > reranker_threshold
                 )

{fabricatio-0.2.13.dev1 → fabricatio-0.2.13.dev3}/python/fabricatio/models/adv_kwargs_types.py

@@ -60,3 +60,4 @@ if find_spec("pymilvus"):
         result_per_query: NotRequired[int]
         tei_endpoint: NotRequired[Optional[str]]
         reranker_threshold: NotRequired[float]
+        filter_expr: NotRequired[str]

{fabricatio-0.2.13.dev1 → fabricatio-0.2.13.dev3}/python/fabricatio/models/extra/aricle_rag.py

@@ -1,6 +1,7 @@
 """A Module containing the article rag models."""
 
 import re
+from itertools import groupby
 from pathlib import Path
 from typing import ClassVar, Dict, List, Optional, Self, Unpack
 
@@ -10,12 +11,13 @@ from fabricatio.models.extra.rag import MilvusDataBase
 from fabricatio.models.generic import AsPrompt
 from fabricatio.models.kwargs_types import ChunkKwargs
 from fabricatio.rust import BibManager, blake3_hash, split_into_chunks
-from fabricatio.utils import ok
+from fabricatio.utils import ok, wrapp_in_block
+from more_itertools.more import first
 from more_itertools.recipes import flatten, unique
 from pydantic import Field
 
 
-class ArticleChunk(MilvusDataBase, AsPrompt):
+class ArticleChunk(MilvusDataBase):
     """The chunk of an article."""
 
     etc_word: ClassVar[str] = "等"
@@ -51,10 +53,9 @@ class ArticleChunk(MilvusDataBase, AsPrompt):
     bibtex_cite_key: str
     """The bibtex cite key of the article"""
 
-    def _as_prompt_inner(self) -> Dict[str, str]:
-        return {
-            f"[[{ok(self._cite_number, 'You need to update cite number first.')}]] reference `{self.article_title}` from {self.as_auther_seq()}": self.chunk
-        }
+    @property
+    def reference_header(self) -> str:
+        return f"[[{ok(self._cite_number, 'You need to update cite number first.')}]] reference `{self.article_title}` from {self.as_auther_seq()}"
 
     @property
     def cite_number(self) -> int:
@@ -204,13 +205,23 @@ class CitationManager(AsPrompt):
 
     def set_cite_number_all(self) -> Self:
         """Set citation numbers for all article chunks."""
-        for i, a in enumerate(self.article_chunks, 1):
-            a.update_cite_number(i)
+        number_mapping = {a.bibtex_cite_key: 0 for a in self.article_chunks}
+
+        for i, k in enumerate(number_mapping.keys()):
+            number_mapping[k] = i
+
+        for a in self.article_chunks:
+            a.update_cite_number(number_mapping[a.bibtex_cite_key])
         return self
 
     def _as_prompt_inner(self) -> Dict[str, str]:
         """Generate prompt inner representation."""
-        return {"References": "\n".join(r.as_prompt() for r in self.article_chunks)}
+        seg = []
+        for k, g in groupby(self.article_chunks, key=lambda a: a.bibtex_cite_key):
+            g = list(g)
+            logger.debug(f"Group [{k}]: {len(g)}")
+            seg.append(wrapp_in_block("\n\n".join(a.chunk for a in g), first(g).reference_header))
+        return {"References": "\n".join(seg)}
 
     def apply(self, string: str) -> str:
         """Apply citation replacements to the input string."""
@@ -261,5 +272,9 @@ class CitationManager(AsPrompt):
 
     def unpack_cite_seq(self, citation_seq: List[int]) -> str:
         """Unpack citation sequence into a string."""
-        chunk_seq = [a for a in self.article_chunks if a.cite_number in citation_seq]
-        return "".join(a.as_typst_cite() for a in chunk_seq)
+        chunk_seq = {a.bibtex_cite_key: a for a in self.article_chunks if a.cite_number in citation_seq}
+        return "".join(a.as_typst_cite() for a in chunk_seq.values())
+
+    def as_milvus_filter_expr(self, blacklist: bool = True) -> str:
+        if blacklist:
+            return " and ".join(f'bibtex_cite_key != "{a.bibtex_cite_key}"' for a in self.article_chunks)

{fabricatio-0.2.13.dev1 → fabricatio-0.2.13.dev3}/python/fabricatio/models/extra/article_base.py

@@ -2,9 +2,12 @@
 
 from abc import ABC
 from enum import StrEnum
+from pathlib import Path
 from typing import ClassVar, Generator, List, Optional, Self, Tuple, Type
 
+from fabricatio.fs import dump_text, safe_text_read
 from fabricatio.fs.readers import extract_sections
+from fabricatio.journal import logger
 from fabricatio.models.generic import (
     AsPrompt,
     Described,
@@ -19,10 +22,12 @@ from fabricatio.models.generic import (
     Titled,
     WordCount,
 )
-from fabricatio.rust import split_out_metadata, to_metadata, word_count
-from fabricatio.utils import fallback_kwargs
+from fabricatio.rust import extract_body, inplace_update, split_out_metadata, to_metadata, word_count
+from fabricatio.utils import fallback_kwargs, ok
 from pydantic import Field
 
+ARTICLE_WRAPPER = "// =-=-=-=-=-=-=-=-=-="
+
 
 class ReferringType(StrEnum):
     """Enumeration of different types of references that can be made in an article."""
@@ -397,3 +402,23 @@ class ArticleBase[T: ChapterBase](FinalizedDumpAble, AsPrompt, FromTypstCode, To
     def avg_wordcount_recursive[S: "ArticleBase"](self: S) -> S:
         """Set all chap, sec, subsec have same word count sum up to be `self.expected_word_count`."""
         return self.avg_chap_wordcount().avg_sec_wordcount().avg_subsec_wordcount()
+
+    def update_article_file(self, file: str | Path) -> Self:
+        """Update the article file."""
+        file = Path(file)
+        string = safe_text_read(file)
+        if updated := inplace_update(string, ARTICLE_WRAPPER, self.to_typst_code()):
+            dump_text(file, updated)
+            logger.success(f"Successfully updated {file.as_posix()}.")
+        else:
+            logger.warning(f"Failed to update {file.as_posix()}. Please make sure there are paired `{ARTICLE_WRAPPER}`")
+        return self
+
+    @classmethod
+    def from_article_file(cls, file: str | Path, title: str) -> Self:
+        """Load article from file."""
+        file = Path(file)
+        string = safe_text_read(file)
+        return cls.from_typst_code(
+            title, ok(extract_body(string, ARTICLE_WRAPPER), "Failed to extract body from file.")
+        )

{fabricatio-0.2.13.dev1 → fabricatio-0.2.13.dev3}/python/fabricatio/models/extra/article_main.py

@@ -53,6 +53,7 @@ class Paragraph(SketchedAble, WordCount, Described):
 
     @property
     def exact_wordcount(self) -> int:
+        """Calculates the exact word count of the content."""
         return word_count(self.content)
 
 

fabricatio-0.2.13.dev3/python/fabricatio/models/extra/article_outline.py

@@ -0,0 +1,45 @@
+"""A module containing the ArticleOutline class, which represents the outline of an academic paper."""
+
+from typing import ClassVar, Dict, Type
+
+from fabricatio.models.extra.article_base import (
+    ArticleBase,
+    ChapterBase,
+    SectionBase,
+    SubSectionBase,
+)
+from fabricatio.models.extra.article_proposal import ArticleProposal
+from fabricatio.models.generic import PersistentAble, WithRef
+
+
+class ArticleSubsectionOutline(SubSectionBase):
+    """Atomic research component specification for academic paper generation."""
+
+
+class ArticleSectionOutline(SectionBase[ArticleSubsectionOutline]):
+    """A slightly more detailed research component specification for academic paper generation, Must contain subsections."""
+
+    child_type: ClassVar[Type[SubSectionBase]] = ArticleSubsectionOutline
+
+
+class ArticleChapterOutline(ChapterBase[ArticleSectionOutline]):
+    """Macro-structural unit implementing standard academic paper organization. Must contain sections."""
+
+    child_type: ClassVar[Type[SectionBase]] = ArticleSectionOutline
+
+
+class ArticleOutline(
+    WithRef[ArticleProposal],
+    PersistentAble,
+    ArticleBase[ArticleChapterOutline],
+):
+    """Outline of an academic paper, containing chapters, sections, subsections."""
+
+    child_type: ClassVar[Type[ChapterBase]] = ArticleChapterOutline
+
+    def _as_prompt_inner(self) -> Dict[str, str]:
+        return {
+            "Original Article Briefing": self.referenced.referenced,
+            "Original Article Proposal": self.referenced.display(),
+            "Original Article Outline": self.display(),
+        }