fabricatio 0.3.14.dev5__tar.gz → 0.3.14.dev7__tar.gz

{fabricatio-0.3.14.dev5 → fabricatio-0.3.14.dev7}/.gitignore

@@ -169,3 +169,4 @@ output/
 templates/customed
 *.typ
 /src/tei.rs
+.python-version

{fabricatio-0.3.14.dev5 → fabricatio-0.3.14.dev7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fabricatio
-Version: 0.3.14.dev5
+Version: 0.3.14.dev7
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: 3.12

{fabricatio-0.3.14.dev5 → fabricatio-0.3.14.dev7}/build.rs

@@ -1,3 +1,6 @@
+use std::fs;
+use std::path::Path;
+
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     tonic_build::configure()
         .out_dir("src")
@@ -5,5 +8,10 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         .build_server(false)
         .type_attribute(".", "#[derive(serde::Serialize,serde::Deserialize)]")
         .compile_protos(&["proto/tei.proto"], &["proto"])
+        .map(|_| {
+            if Path::new("src/tei.v1.rs").exists() {
+                fs::rename("src/tei.v1.rs", "src/tei.rs").expect("Failed to rename file")
+            }
+        })
         .map_err(|e| e.into())
 }

{fabricatio-0.3.14.dev5 → fabricatio-0.3.14.dev7}/proto/tei.proto

@@ -1,6 +1,6 @@
 syntax = "proto3";
 
-package tei;
+package tei.v1;
 
 service Info {
     rpc Info (InfoRequest) returns (InfoResponse) {

{fabricatio-0.3.14.dev5 → fabricatio-0.3.14.dev7}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "fabricatio"
-version = "0.3.14-dev5"
+version = "0.3.14-dev7"
 description = "A LLM multi-agent framework."
 readme = "README.md"
 license = { file = "LICENSE" }

{fabricatio-0.3.14.dev5 → fabricatio-0.3.14.dev7}/python/fabricatio/actions/article_rag.py

@@ -1,10 +1,10 @@
 """A module for writing articles using RAG (Retrieval-Augmented Generation) capabilities."""
 
 from asyncio import gather
-from pathlib import Path
-from typing import ClassVar, List, Optional
 
+from pathlib import Path
 from pydantic import Field, PositiveInt
+from typing import ClassVar, List, Optional
 
 from fabricatio.capabilities.advanced_rag import AdvancedRAG
 from fabricatio.capabilities.censor import Censor
@@ -21,10 +21,7 @@ from fabricatio.models.extra.rule import RuleSet
 from fabricatio.models.kwargs_types import ChooseKwargs, LLMKwargs
 from fabricatio.rust import (
     BibManager,
-    convert_all_block_tex,
-    convert_all_inline_tex,
-    convert_to_block_formula,
-    convert_to_inline_formula,
+    convert_all_tex_math,
     fix_misplaced_labels,
 )
 from fabricatio.utils import ok
@@ -78,11 +75,11 @@ class WriteArticleContentRAG(Action, Extract, AdvancedRAG):
     tei_endpoint: Optional[str] = None
 
     async def _execute(
-        self,
-        article_outline: ArticleOutline,
-        collection_name: Optional[str] = None,
-        supervisor: Optional[bool] = None,
-        **cxt,
+            self,
+            article_outline: ArticleOutline,
+            collection_name: Optional[str] = None,
+            supervisor: Optional[bool] = None,
+            **cxt,
     ) -> Article:
         article = Article.from_outline(article_outline).update_ref(article_outline)
         self.target_collection = collection_name or self.safe_target_collection
@@ -103,12 +100,12 @@ class WriteArticleContentRAG(Action, Extract, AdvancedRAG):
         "questionary", "`questionary` is required for supervisor mode, please install it by `fabricatio[qa]`"
     )
     async def _supervisor_inner(
-        self,
-        article: Article,
-        article_outline: ArticleOutline,
-        chap: ArticleChapter,
-        sec: ArticleSection,
-        subsec: ArticleSubsection,
+            self,
+            article: Article,
+            article_outline: ArticleOutline,
+            chap: ArticleChapter,
+            sec: ArticleSection,
+            subsec: ArticleSubsection,
     ) -> ArticleSubsection:
         from questionary import confirm, text
         from rich import print as r_print
@@ -129,20 +126,19 @@ class WriteArticleContentRAG(Action, Extract, AdvancedRAG):
                 raw_paras = edt
 
             raw_paras = fix_misplaced_labels(raw_paras)
-            raw_paras = convert_all_inline_tex(raw_paras)
-            raw_paras = convert_all_block_tex(raw_paras)
+            raw_paras = convert_all_tex_math(raw_paras)
 
             r_print(raw_paras)
 
         return await self.extract_new_subsec(subsec, raw_paras, cm)
 
     async def _inner(
-        self,
-        article: Article,
-        article_outline: ArticleOutline,
-        chap: ArticleChapter,
-        sec: ArticleSection,
-        subsec: ArticleSubsection,
+            self,
+            article: Article,
+            article_outline: ArticleOutline,
+            chap: ArticleChapter,
+            sec: ArticleSection,
+            subsec: ArticleSubsection,
     ) -> ArticleSubsection:
         cm = CitationManager()
 
@@ -153,13 +149,12 @@ class WriteArticleContentRAG(Action, Extract, AdvancedRAG):
         raw_paras = "\n".join(p for p in raw_paras.splitlines() if p and not p.endswith("**") and not p.startswith("#"))
 
         raw_paras = fix_misplaced_labels(raw_paras)
-        raw_paras = convert_all_inline_tex(raw_paras)
-        raw_paras = convert_all_block_tex(raw_paras)
+        raw_paras = convert_all_tex_math(raw_paras)
 
         return await self.extract_new_subsec(subsec, raw_paras, cm)
 
     async def extract_new_subsec(
-        self, subsec: ArticleSubsection, raw_paras: str, cm: CitationManager
+            self, subsec: ArticleSubsection, raw_paras: str, cm: CitationManager
     ) -> ArticleSubsection:
         """Extract the new subsec."""
         new_subsec = ok(
@@ -182,14 +177,14 @@ class WriteArticleContentRAG(Action, Extract, AdvancedRAG):
         return subsec
 
     async def write_raw(
-        self,
-        article: Article,
-        article_outline: ArticleOutline,
-        chap: ArticleChapter,
-        sec: ArticleSection,
-        subsec: ArticleSubsection,
-        cm: CitationManager,
-        extra_instruction: str = "",
+            self,
+            article: Article,
+            article_outline: ArticleOutline,
+            chap: ArticleChapter,
+            sec: ArticleSection,
+            subsec: ArticleSubsection,
+            cm: CitationManager,
+            extra_instruction: str = "",
     ) -> str:
         """Write the raw paragraphs of the subsec."""
         return await self.aask(
@@ -205,14 +200,14 @@ class WriteArticleContentRAG(Action, Extract, AdvancedRAG):
         )
 
     async def search_database(
-        self,
-        article: Article,
-        article_outline: ArticleOutline,
-        chap: ArticleChapter,
-        sec: ArticleSection,
-        subsec: ArticleSubsection,
-        cm: CitationManager,
-        extra_instruction: str = "",
+            self,
+            article: Article,
+            article_outline: ArticleOutline,
+            chap: ArticleChapter,
+            sec: ArticleSection,
+            subsec: ArticleSubsection,
+            cm: CitationManager,
+            extra_instruction: str = "",
     ) -> None:
         """Search database for related references."""
         search_req = (
@@ -261,8 +256,6 @@ class ArticleConsultRAG(Action, AdvancedRAG):
         from questionary import confirm, text
         from rich import print as r_print
 
-        from fabricatio.rust import convert_all_block_tex, convert_all_inline_tex, fix_misplaced_labels
-
         self.target_collection = collection_name or self.safe_target_collection
 
         cm = CitationManager()
@@ -272,8 +265,7 @@ class ArticleConsultRAG(Action, AdvancedRAG):
             if await confirm("Empty the cm?").ask_async():
                 cm.empty()
 
-            req = convert_to_block_formula(req)
-            req = convert_to_inline_formula(req)
+            req = convert_all_tex_math(req)
 
             await self.clued_search(
                 req,
@@ -289,8 +281,7 @@ class ArticleConsultRAG(Action, AdvancedRAG):
             ret = await self.aask(f"{cm.as_prompt()}\n{self.req}\n{req}")
 
             ret = fix_misplaced_labels(ret)
-            ret = convert_all_inline_tex(ret)
-            ret = convert_all_block_tex(ret)
+            ret = convert_all_tex_math(ret)
             ret = cm.apply(ret)
 
             r_print(ret)
@@ -321,12 +312,12 @@ class TweakArticleRAG(Action, RAG, Censor):
     """The limit of references to be retrieved"""
 
     async def _execute(
-        self,
-        article: Article,
-        collection_name: str = "article_essence",
-        twk_rag_ruleset: Optional[RuleSet] = None,
-        parallel: bool = False,
-        **cxt,
+            self,
+            article: Article,
+            collection_name: str = "article_essence",
+            twk_rag_ruleset: Optional[RuleSet] = None,
+            parallel: bool = False,
+            **cxt,
     ) -> Article:
         """Write an article based on the provided outline.
 
@@ -381,10 +372,10 @@ class TweakArticleRAG(Action, RAG, Censor):
             subsec,
             ruleset=ruleset,
             reference=f"{'\n\n'.join(d.display() for d in await self.aretrieve(refind_q, document_model=ArticleEssence, max_accepted=self.ref_limit))}\n\n"
-            f"You can use Reference above to rewrite the `{subsec.__class__.__name__}`.\n"
-            f"You should Always use `{subsec.language}` as written language, "
-            f"which is the original language of the `{subsec.title}`. "
-            f"since rewrite a `{subsec.__class__.__name__}` in a different language is usually a bad choice",
+                      f"You can use Reference above to rewrite the `{subsec.__class__.__name__}`.\n"
+                      f"You should Always use `{subsec.language}` as written language, "
+                      f"which is the original language of the `{subsec.title}`. "
+                      f"since rewrite a `{subsec.__class__.__name__}` in a different language is usually a bad choice",
         )
 
 
@@ -399,12 +390,12 @@ class ChunkArticle(Action):
     """The maximum overlapping rate between chunks."""
 
     async def _execute(
-        self,
-        article_path: str | Path,
-        bib_manager: BibManager,
-        max_chunk_size: Optional[int] = None,
-        max_overlapping_rate: Optional[float] = None,
-        **_,
+            self,
+            article_path: str | Path,
+            bib_manager: BibManager,
+            max_chunk_size: Optional[int] = None,
+            max_overlapping_rate: Optional[float] = None,
+            **_,
     ) -> List[ArticleChunk]:
         return ArticleChunk.from_file(
             article_path,

{fabricatio-0.3.14.dev5 → fabricatio-0.3.14.dev7}/python/fabricatio/capabilities/rag.py

@@ -51,16 +51,16 @@ class RAG(EmbeddingUsage, ABC):
         return self._client
 
     def init_client(
-        self,
-        milvus_uri: Optional[str] = None,
-        milvus_token: Optional[str] = None,
-        milvus_timeout: Optional[float] = None,
+            self,
+            milvus_uri: Optional[str] = None,
+            milvus_token: Optional[str] = None,
+            milvus_timeout: Optional[float] = None,
     ) -> Self:
         """Initialize the Milvus client."""
         self._client = create_client(
             uri=milvus_uri or ok(self.milvus_uri or CONFIG.rag.milvus_uri),
             token=milvus_token
-            or (token.get_secret_value() if (token := (self.milvus_token or CONFIG.rag.milvus_token)) else ""),
+                  or (token.get_secret_value() if (token := (self.milvus_token or CONFIG.rag.milvus_token)) else ""),
             timeout=milvus_timeout or self.milvus_timeout or CONFIG.rag.milvus_timeout,
         )
         return self
@@ -74,7 +74,7 @@ class RAG(EmbeddingUsage, ABC):
         return self
 
     def view(
-        self, collection_name: Optional[str], create: bool = False, **kwargs: Unpack[CollectionConfigKwargs]
+            self, collection_name: Optional[str], create: bool = False, **kwargs: Unpack[CollectionConfigKwargs]
     ) -> Self:
         """View the specified collection.
 
@@ -116,7 +116,7 @@ class RAG(EmbeddingUsage, ABC):
         return ok(self.target_collection, "No collection is being viewed. Have you called `self.view()`?")
 
     async def add_document[D: MilvusDataBase](
-        self, data: List[D] | D, collection_name: Optional[str] = None, flush: bool = False
+            self, data: List[D] | D, collection_name: Optional[str] = None, flush: bool = False
     ) -> Self:
         """Adds a document to the specified collection.
 
@@ -143,15 +143,15 @@ class RAG(EmbeddingUsage, ABC):
         return self
 
     async def afetch_document[D: MilvusDataBase](
-        self,
-        query: List[str],
-        document_model: Type[D],
-        collection_name: Optional[str] = None,
-        similarity_threshold: float = 0.37,
-        result_per_query: int = 10,
-        tei_endpoint: Optional[str] = None,
-        reranker_threshold: float = 0.7,
-        filter_expr: str = "",
+            self,
+            query: List[str],
+            document_model: Type[D],
+            collection_name: Optional[str] = None,
+            similarity_threshold: float = 0.37,
+            result_per_query: int = 10,
+            tei_endpoint: Optional[str] = None,
+            reranker_threshold: float = 0.7,
+            filter_expr: str = "",
     ) -> List[D]:
         """Asynchronously fetches documents from a Milvus database based on input vectors.
 
@@ -192,7 +192,8 @@ class RAG(EmbeddingUsage, ABC):
                 retrieved_id.update(res["id"] for res in g)
                 if not models:
                     continue
-                rank_scores = await reranker.arerank(q, [m.prepare_vectorization() for m in models], truncate=True)
+                rank_scores = await reranker.arerank(q, [m.prepare_vectorization() for m in models], truncate=True,
+                                                     truncation_direction="Left")
                 raw_result.extend((models[idx], scr) for (idx, scr) in rank_scores if scr > reranker_threshold)
 
             raw_result_sorted = sorted(raw_result, key=lambda x: x[1], reverse=True)
@@ -214,11 +215,11 @@ class RAG(EmbeddingUsage, ABC):
         return document_model.from_sequence(resp)
 
     async def aretrieve[D: MilvusDataBase](
-        self,
-        query: List[str] | str,
-        document_model: Type[D],
-        max_accepted: int = 20,
-        **kwargs: Unpack[FetchKwargs],
+            self,
+            query: List[str] | str,
+            document_model: Type[D],
+            max_accepted: int = 20,
+            **kwargs: Unpack[FetchKwargs],
     ) -> List[D]:
         """Retrieve data from the collection.
 
@@ -235,15 +236,15 @@ class RAG(EmbeddingUsage, ABC):
             query = [query]
 
         return (
-            await self.afetch_document(
-                query=query,
-                document_model=document_model,
-                **kwargs,
-            )
-        )[:max_accepted]
+                   await self.afetch_document(
+                       query=query,
+                       document_model=document_model,
+                       **kwargs,
+                   )
+               )[:max_accepted]
 
     async def arefined_query(
-        self, question: List[str] | str, **kwargs: Unpack[ChooseKwargs[Optional[List[str]]]]
+            self, question: List[str] | str, **kwargs: Unpack[ChooseKwargs[Optional[List[str]]]]
     ) -> Optional[List[str]]:
         """Refines the given question using a template.
 

{fabricatio-0.3.14.dev5 → fabricatio-0.3.14.dev7}/python/fabricatio/models/extra/aricle_rag.py

@@ -1,8 +1,13 @@
 """A Module containing the article rag models."""
 
-import re
 from itertools import groupby
+
+import re
+from dataclasses import dataclass, field
+from more_itertools.more import first
+from more_itertools.recipes import flatten, unique
 from pathlib import Path
+from pydantic import Field
 from typing import ClassVar, Dict, List, Optional, Self, Unpack
 
 from fabricatio.fs import safe_text_read
@@ -12,9 +17,6 @@ from fabricatio.models.generic import AsPrompt
 from fabricatio.models.kwargs_types import ChunkKwargs
 from fabricatio.rust import BibManager, blake3_hash, split_into_chunks
 from fabricatio.utils import ok, wrapp_in_block
-from more_itertools.more import first
-from more_itertools.recipes import flatten, unique
-from pydantic import Field
 
 
 class ArticleChunk(MilvusDataBase):
@@ -68,7 +70,7 @@ class ArticleChunk(MilvusDataBase):
 
     @classmethod
     def from_file[P: str | Path](
-        cls, path: P | List[P], bib_mgr: BibManager, **kwargs: Unpack[ChunkKwargs]
+            cls, path: P | List[P], bib_mgr: BibManager, **kwargs: Unpack[ChunkKwargs]
     ) -> List[Self]:
         """Load the article chunks from the file."""
         if isinstance(path, list):
@@ -85,9 +87,9 @@ class ArticleChunk(MilvusDataBase):
         title_seg = path.stem.split(" - ").pop()
 
         key = (
-            bib_mgr.get_cite_key_by_title(title_seg)
-            or bib_mgr.get_cite_key_by_title_fuzzy(title_seg)
-            or bib_mgr.get_cite_key_fuzzy(path.stem)
+                bib_mgr.get_cite_key_by_title(title_seg)
+                or bib_mgr.get_cite_key_by_title_fuzzy(title_seg)
+                or bib_mgr.get_cite_key_fuzzy(path.stem)
         )
         if key is None:
             logger.warning(f"no cite key found for {path.as_posix()}, skip.")
@@ -165,10 +167,11 @@ class ArticleChunk(MilvusDataBase):
         return self
 
 
+@dataclass
 class CitationManager(AsPrompt):
     """Citation manager."""
 
-    article_chunks: List[ArticleChunk] = Field(default_factory=list)
+    article_chunks: List[ArticleChunk] = field(default_factory=list)
     """Article chunks."""
 
     pat: str = r"(\[\[([\d\s,-]*)]])"
@@ -179,7 +182,7 @@ class CitationManager(AsPrompt):
     """Separator for abbreviated citation numbers."""
 
     def update_chunks(
-        self, article_chunks: List[ArticleChunk], set_cite_number: bool = True, dedup: bool = True
+            self, article_chunks: List[ArticleChunk], set_cite_number: bool = True, dedup: bool = True
     ) -> Self:
         """Update article chunks."""
         self.article_chunks.clear()

{fabricatio-0.3.14.dev5 → fabricatio-0.3.14.dev7}/python/fabricatio/models/extra/article_main.py

@@ -1,5 +1,6 @@
 """ArticleBase and ArticleSubsection classes for managing hierarchical document components."""
 
+from pydantic import Field, NonNegativeInt
 from typing import ClassVar, Dict, Generator, List, Self, Tuple, Type, override
 
 from fabricatio.capabilities.persist import PersistentAble
@@ -19,13 +20,11 @@ from fabricatio.models.extra.article_outline import (
 )
 from fabricatio.models.generic import Described, SequencePatch, SketchedAble, WithRef, WordCount
 from fabricatio.rust import (
-    convert_all_block_tex,
-    convert_all_inline_tex,
+    convert_all_tex_math,
     fix_misplaced_labels,
     split_out_metadata,
     word_count,
 )
-from pydantic import Field, NonNegativeInt
 
 PARAGRAPH_SEP = "// - - -"
 
@@ -82,8 +81,8 @@ class ArticleSubsection(SubSectionBase):
         if len(self.paragraphs) == 0:
             summary += f"`{self.__class__.__name__}` titled `{self.title}` have no paragraphs, You should add some!\n"
         if (
-            abs((wc := self.word_count) - self.expected_word_count) / self.expected_word_count
-            > self._max_word_count_deviation
+                abs((wc := self.word_count) - self.expected_word_count) / self.expected_word_count
+                > self._max_word_count_deviation
         ):
             summary += f"`{self.__class__.__name__}` titled `{self.title}` have {wc} words, expected {self.expected_word_count} words!"
 
@@ -155,15 +154,13 @@ class Article(
         if descriptions:
             for a in self.iter_dfs():
                 a.description = fix_misplaced_labels(a.description)
-                a.description = convert_all_inline_tex(a.description)
-                a.description = convert_all_block_tex(a.description)
+                a.description = convert_all_tex_math(a.description)
 
         if paragraphs:
             for _, _, subsec in self.iter_subsections():
                 for p in subsec.paragraphs:
                     p.content = fix_misplaced_labels(p.content)
-                    p.content = convert_all_inline_tex(p.content)
-                    p.content = convert_all_block_tex(p.content)
+                    p.content = convert_all_tex_math(p.content)
         return self
 
     @override

{fabricatio-0.3.14.dev5 → fabricatio-0.3.14.dev7}/python/fabricatio/rust.pyi

@@ -325,9 +325,10 @@ def tex_to_typst(string: str) -> str:
 
 
 def convert_all_tex_math(string: str) -> str:
-    """Unified function to convert all supported TeX math expressions in a string to Typst format.
+    r"""Unified function to convert all supported TeX math expressions in a string to Typst format.
 
     Handles $...$, $$...$$, \\(...\\), and \\[...\\]
+
     Args:
         string: The input string containing TeX math expressions.
 
@@ -348,7 +349,7 @@ def fix_misplaced_labels(string: str) -> str:
 
 
 def comment(string: str) -> str:
-    """Add comment to the string.
+    r"""Add comment to the string.
 
     Args:
         string: The input string to which comments will be added.