fabricatio 0.2.10.dev0__cp312-cp312-win_amd64.whl → 0.2.11.dev0__cp312-cp312-win_amd64.whl

fabricatio/decorators.py

@@ -8,14 +8,34 @@ from shutil import which
 from types import ModuleType
 from typing import Callable, List, Optional
 
-from questionary import confirm
-
 from fabricatio.config import configs
 from fabricatio.journal import logger
 
 
+def precheck_package[**P, R](package_name: str, msg: str) -> Callable[[Callable[P, R]], Callable[P, R]]:
+    """Check if a package exists in the current environment.
+
+    Args:
+        package_name (str): The name of the package to check.
+        msg (str): The message to display if the package is not found.
+
+    Returns:
+        bool: True if the package exists, False otherwise.
+    """
+
+    def _wrapper(func: Callable[P, R]) -> Callable[P, R]:
+        def _inner(*args: P.args, **kwargs: P.kwargs) -> R:
+            if find_spec(package_name):
+                return func(*args, **kwargs)
+            raise RuntimeError(msg)
+
+        return _inner
+
+    return _wrapper
+
+
 def depend_on_external_cmd[**P, R](
-    bin_name: str, install_tip: Optional[str], homepage: Optional[str] = None
+        bin_name: str, install_tip: Optional[str], homepage: Optional[str] = None
 ) -> Callable[[Callable[P, R]], Callable[P, R]]:
     """Decorator to check for the presence of an external command.
 
@@ -68,6 +88,8 @@ def logging_execution_info[**P, R](func: Callable[P, R]) -> Callable[P, R]:
     return _wrapper
 
 
+@precheck_package("questionary",
+                  "'questionary' is required to run this function. Have you installed `fabricatio[qa]`?.")
 def confirm_to_execute[**P, R](func: Callable[P, R]) -> Callable[P, Optional[R]] | Callable[P, R]:
     """Decorator to confirm before executing a function.
 
@@ -80,14 +102,15 @@ def confirm_to_execute[**P, R](func: Callable[P, R]) -> Callable[P, Optional[R]]
     if not configs.general.confirm_on_ops:
         # Skip confirmation if the configuration is set to False
         return func
+    from questionary import confirm
 
     if iscoroutinefunction(func):
 
         @wraps(func)
         async def _wrapper(*args: P.args, **kwargs: P.kwargs) -> Optional[R]:
             if await confirm(
-                f"Are you sure to execute function: {func.__name__}{signature(func)} \n📦 Args:{args}\n🔑 Kwargs:{kwargs}\n",
-                instruction="Please input [Yes/No] to proceed (default: Yes):",
+                    f"Are you sure to execute function: {func.__name__}{signature(func)} \n📦 Args:{args}\n🔑 Kwargs:{kwargs}\n",
+                    instruction="Please input [Yes/No] to proceed (default: Yes):",
             ).ask_async():
                 return await func(*args, **kwargs)
             logger.warning(f"Function: {func.__name__}{signature(func)} canceled by user.")
@@ -98,8 +121,8 @@ def confirm_to_execute[**P, R](func: Callable[P, R]) -> Callable[P, Optional[R]]
         @wraps(func)
         def _wrapper(*args: P.args, **kwargs: P.kwargs) -> Optional[R]:
             if confirm(
-                f"Are you sure to execute function: {func.__name__}{signature(func)} \n📦 Args:{args}\n��� Kwargs:{kwargs}\n",
-                instruction="Please input [Yes/No] to proceed (default: Yes):",
+                    f"Are you sure to execute function: {func.__name__}{signature(func)} \n📦 Args:{args}\n��� Kwargs:{kwargs}\n",
+                    instruction="Please input [Yes/No] to proceed (default: Yes):",
             ).ask():
                 return func(*args, **kwargs)
             logger.warning(f"Function: {func.__name__}{signature(func)} canceled by user.")
@@ -192,7 +215,6 @@ def logging_exec_time[**P, R](func: Callable[P, R]) -> Callable[P, R]:
     from time import time
 
     if iscoroutinefunction(func):
-
         @wraps(func)
         async def _async_wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
             start_time = time()
@@ -210,25 +232,3 @@ def logging_exec_time[**P, R](func: Callable[P, R]) -> Callable[P, R]:
         return result
 
     return _wrapper
-
-
-def precheck_package[**P, R](package_name: str, msg: str) -> Callable[[Callable[P, R]], Callable[P, R]]:
-    """Check if a package exists in the current environment.
-
-    Args:
-        package_name (str): The name of the package to check.
-        msg (str): The message to display if the package is not found.
-
-    Returns:
-        bool: True if the package exists, False otherwise.
-    """
-
-    def _wrapper(func: Callable[P, R]) -> Callable[P, R]:
-        def _inner(*args: P.args, **kwargs: P.kwargs) -> R:
-            if find_spec(package_name):
-                return func(*args, **kwargs)
-            raise RuntimeError(msg)
-
-        return _inner
-
-    return _wrapper

fabricatio/fs/__init__.py

@@ -1,5 +1,7 @@
 """FileSystem manipulation module for Fabricatio."""
+from importlib.util import find_spec
 
+from fabricatio.config import configs
 from fabricatio.fs.curd import (
     absolute_path,
     copy_file,
@@ -11,10 +13,9 @@ from fabricatio.fs.curd import (
     move_file,
     tree,
 )
-from fabricatio.fs.readers import MAGIKA, safe_json_read, safe_text_read
+from fabricatio.fs.readers import safe_json_read, safe_text_read
 
 __all__ = [
-    "MAGIKA",
     "absolute_path",
     "copy_file",
     "create_directory",
@@ -27,3 +28,9 @@ __all__ = [
     "safe_text_read",
     "tree",
 ]
+
+if find_spec("magika"):
+    from magika import Magika
+
+    MAGIKA = Magika(model_dir=configs.magika.model_dir)
+    __all__ += ["MAGIKA"]

fabricatio/fs/readers.py

@@ -1,17 +1,13 @@
 """Filesystem readers for Fabricatio."""
 
+import re
 from pathlib import Path
 from typing import Dict, List, Tuple
 
-import orjson
-import regex
-from magika import Magika
+import ujson
 
-from fabricatio.config import configs
 from fabricatio.journal import logger
 
-MAGIKA = Magika(model_dir=configs.magika.model_dir)
-
 
 def safe_text_read(path: Path | str) -> str:
     """Safely read the text from a file.
@@ -41,8 +37,8 @@ def safe_json_read(path: Path | str) -> Dict:
     """
     path = Path(path)
     try:
-        return orjson.loads(path.read_text(encoding="utf-8"))
-    except (orjson.JSONDecodeError, IsADirectoryError, FileNotFoundError) as e:
+        return ujson.loads(path.read_text(encoding="utf-8"))
+    except (ujson.JSONDecodeError, IsADirectoryError, FileNotFoundError) as e:
         logger.error(f"Failed to read file {path}: {e!s}")
         return {}
 
@@ -58,8 +54,8 @@ def extract_sections(string: str, level: int, section_char: str = "#") -> List[T
     Returns:
         List[Tuple[str, str]]: List of (header_text, section_content) tuples
     """
-    return regex.findall(
+    return re.findall(
         r"^%s{%d}\s+(.+?)\n((?:(?!^%s{%d}\s).|\n)*)" % (section_char, level, section_char, level),
         string,
-        regex.MULTILINE,
+        re.MULTILINE,
     )

fabricatio/models/adv_kwargs_types.py

@@ -1,10 +1,9 @@
 """A module containing kwargs types for content correction and checking operations."""
 
 from importlib.util import find_spec
-from typing import Required, Type, TypedDict
+from typing import NotRequired, TypedDict
 
 from fabricatio.models.extra.problem import Improvement
-from fabricatio.models.extra.rag import MilvusDataBase
 from fabricatio.models.extra.rule import RuleSet
 from fabricatio.models.generic import SketchedAble
 from fabricatio.models.kwargs_types import ReferencedKwargs
@@ -49,19 +48,13 @@ if find_spec("pymilvus"):
         schema: CollectionSchema | None
         index_params: IndexParams | None
 
-    class FetchKwargs[D: MilvusDataBase](TypedDict, total=False):
+    class FetchKwargs(TypedDict):
         """Arguments for fetching data from vector collections.
 
         Controls how data is retrieved from vector databases, including filtering
         and result limiting parameters.
         """
 
-        document_model: Required[Type[D]]
-        collection_name: str | None
-        similarity_threshold: float
-        result_per_query: int
-
-    class RetrievalKwargs(FetchKwargs, total=False):
-        """Arguments for retrieval operations."""
-
-        final_limit: int
+        collection_name: NotRequired[str | None]
+        similarity_threshold: NotRequired[float]
+        result_per_query: NotRequired[int]

fabricatio/models/extra/aricle_rag.py

@@ -0,0 +1,235 @@
+"""A Module containing the article rag models."""
+
+import re
+from pathlib import Path
+from typing import ClassVar, Dict, List, Optional, Self, Unpack
+
+from fabricatio.fs import safe_text_read
+from fabricatio.journal import logger
+from fabricatio.models.extra.rag import MilvusDataBase
+from fabricatio.models.generic import AsPrompt
+from fabricatio.models.kwargs_types import ChunkKwargs
+from fabricatio.rust import BibManager, is_chinese, split_into_chunks
+from fabricatio.utils import ok
+from more_itertools.recipes import flatten, unique
+from pydantic import Field
+
+
+class ArticleChunk(MilvusDataBase, AsPrompt):
+    """The chunk of an article."""
+
+    etc_word: ClassVar[str] = "等"
+    and_word: ClassVar[str] = "与"
+    _cite_number: Optional[int] = None
+
+    head_split: ClassVar[List[str]] = [
+        "引 言",
+        "引言",
+        "绪 论",
+        "绪论",
+        "前言",
+        "INTRODUCTION",
+        "Introduction",
+    ]
+    tail_split: ClassVar[List[str]] = [
+        "参 考 文 献",
+        "参  考  文  献",
+        "参考文献",
+        "REFERENCES",
+        "References",
+        "Bibliography",
+        "Reference",
+    ]
+    chunk: str
+    """The segment of the article"""
+    year: int
+    """The year of the article"""
+    authors: List[str] = Field(default_factory=list)
+    """The authors of the article"""
+    article_title: str
+    """The title of the article"""
+    bibtex_cite_key: str
+    """The bibtex cite key of the article"""
+
+    def _as_prompt_inner(self) -> Dict[str, str]:
+        return {
+            f"[[{ok(self._cite_number, 'You need to update cite number first.')}]] reference `{self.article_title}`": self.chunk
+        }
+
+    @property
+    def cite_number(self) -> int:
+        """Get the cite number."""
+        return ok(self._cite_number, "cite number not set")
+
+    def _prepare_vectorization_inner(self) -> str:
+        return self.chunk
+
+    @classmethod
+    def from_file[P: str | Path](
+        cls, path: P | List[P], bib_mgr: BibManager, **kwargs: Unpack[ChunkKwargs]
+    ) -> List[Self]:
+        """Load the article chunks from the file."""
+        if isinstance(path, list):
+            result = list(flatten(cls._from_file_inner(p, bib_mgr, **kwargs) for p in path))
+            logger.debug(f"Number of chunks created from list of files: {len(result)}")
+            return result
+
+        return cls._from_file_inner(path, bib_mgr, **kwargs)
+
+    @classmethod
+    def _from_file_inner(cls, path: str | Path, bib_mgr: BibManager, **kwargs: Unpack[ChunkKwargs]) -> List[Self]:
+        path = Path(path)
+
+        title_seg = path.stem.split(" - ").pop()
+
+        key = (
+            bib_mgr.get_cite_key_by_title(title_seg)
+            or bib_mgr.get_cite_key_by_title_fuzzy(title_seg)
+            or bib_mgr.get_cite_key_fuzzy(path.stem)
+        )
+        if key is None:
+            logger.warning(f"no cite key found for {path.as_posix()}, skip.")
+            return []
+        authors = ok(bib_mgr.get_author_by_key(key), f"no author found for {key}")
+        year = ok(bib_mgr.get_year_by_key(key), f"no year found for {key}")
+        article_title = ok(bib_mgr.get_title_by_key(key), f"no title found for {key}")
+
+        result = [
+            cls(chunk=c, year=year, authors=authors, article_title=article_title, bibtex_cite_key=key)
+            for c in split_into_chunks(cls.purge_numeric_citation(cls.strip(safe_text_read(path))), **kwargs)
+        ]
+
+        logger.debug(f"Number of chunks created from file {path.as_posix()}: {len(result)}")
+        return result
+
+    @classmethod
+    def strip(cls, string: str) -> str:
+        """Strip the head and tail of the string."""
+        logger.debug(f"String length before strip: {(original := len(string))}")
+        for split in (s for s in cls.head_split if s in string):
+            logger.debug(f"Strip head using {split}")
+            parts = string.split(split)
+            string = split.join(parts[1:]) if len(parts) > 1 else parts[0]
+            break
+        logger.debug(
+            f"String length after head strip: {(stripped_len := len(string))}, decreased by {(d := original - stripped_len)}"
+        )
+        if not d:
+            logger.warning("No decrease at head strip, which is might be abnormal.")
+        for split in (s for s in cls.tail_split if s in string):
+            logger.debug(f"Strip tail using {split}")
+            parts = string.split(split)
+            string = split.join(parts[:-1]) if len(parts) > 1 else parts[0]
+            break
+        logger.debug(f"String length after tail strip: {len(string)}, decreased by {(d := stripped_len - len(string))}")
+        if not d:
+            logger.warning("No decrease at tail strip, which is might be abnormal.")
+
+        return string
+
+    def as_typst_cite(self) -> str:
+        """As typst cite."""
+        return f"#cite(<{self.bibtex_cite_key}>)"
+
+    @staticmethod
+    def purge_numeric_citation(string: str) -> str:
+        """Purge numeric citation."""
+        import re
+
+        return re.sub(r"\[[\d\s,\\~–-]+]", "", string)  # noqa: RUF001
+
+    @property
+    def auther_firstnames(self) -> List[str]:
+        """Get the first name of the authors."""
+        ret = []
+        for n in self.authors:
+            if is_chinese(n):
+                ret.append(n[0])
+            else:
+                ret.append(n.split()[-1])
+        return ret
+
+    def as_auther_seq(self) -> str:
+        """Get the auther sequence."""
+        match len(self.authors):
+            case 0:
+                raise ValueError("No authors found")
+            case 1:
+                return f"（{self.auther_firstnames[0]}，{self.year}）{self.as_typst_cite()}"
+            case 2:
+                return f"（{self.auther_firstnames[0]}{self.and_word}{self.auther_firstnames[1]}，{self.year}）{self.as_typst_cite()}"
+            case 3:
+                return f"（{self.auther_firstnames[0]}，{self.auther_firstnames[1]}{self.and_word}{self.auther_firstnames[2]}，{self.year}）{self.as_typst_cite()}"
+            case _:
+                return f"（{self.auther_firstnames[0]}，{self.auther_firstnames[1]}{self.and_word}{self.auther_firstnames[2]}{self.etc_word}，{self.year}）{self.as_typst_cite()}"
+
+    def update_cite_number(self, cite_number: int) -> Self:
+        """Update the cite number."""
+        self._cite_number = cite_number
+        return self
+
+
+class CitationManager(AsPrompt):
+    """Citation manager."""
+
+    article_chunks: List[ArticleChunk] = Field(default_factory=list)
+    """Article chunks."""
+
+    pat: str = r"\[\[([\d\s,-]*)]]"
+    """Regex pattern to match citations."""
+    sep: str = ","
+    """Separator for citation numbers."""
+    abbr_sep: str = "-"
+    """Separator for abbreviated citation numbers."""
+
+    def update_chunks(self, article_chunks: List[ArticleChunk], set_cite_number: bool = True) -> Self:
+        """Update article chunks."""
+        self.article_chunks.clear()
+        self.article_chunks.extend(article_chunks)
+        if set_cite_number:
+            self.set_cite_number_all()
+        return self
+
+    def set_cite_number_all(self) -> Self:
+        """Set citation numbers for all article chunks."""
+        for i, a in enumerate(self.article_chunks, 1):
+            a.update_cite_number(i)
+        return self
+
+    def _as_prompt_inner(self) -> Dict[str, str]:
+        """Generate prompt inner representation."""
+        return {"References": "\n".join(r.as_prompt() for r in self.article_chunks)}
+
+    def apply(self, string: str) -> str:
+        """Apply citation replacements to the input string."""
+        matches = re.findall(self.pat, string)
+
+        for m in matches:
+            notations = self.convert_to_numeric_notations(m)
+
+            citation_number_seq = list(flatten(self.decode_expr(n) for n in notations))
+            dedup = self.deduplicate_citation(citation_number_seq)
+            string.replace(m, self.unpack_cite_seq(dedup))
+        return string
+
+    def decode_expr(self, string: str) -> List[int]:
+        """Decode citation expression into a list of integers."""
+        if self.abbr_sep in string:
+            start, end = string.split(self.abbr_sep)
+            return list(range(int(start), int(end) + 1))
+        return [int(string)]
+
+    def convert_to_numeric_notations(self, string: str) -> List[str]:
+        """Convert citation string into numeric notations."""
+        return [s.strip() for s in string.split(self.sep)]
+
+    def deduplicate_citation(self, citation_seq: List[int]) -> List[int]:
+        """Deduplicate citation sequence."""
+        chunk_seq = [a for a in self.article_chunks if a.cite_number in citation_seq]
+        deduped = unique(chunk_seq, lambda a: a.cite_number)
+        return [a.cite_number for a in deduped]
+
+    def unpack_cite_seq(self, citation_seq: List[int]) -> str:
+        """Unpack citation sequence into a string."""
+        chunk_seq = [a for a in self.article_chunks if a.cite_number in citation_seq]
+        return "".join(a.as_typst_cite() for a in chunk_seq)

fabricatio/models/extra/article_essence.py

@@ -1,8 +1,9 @@
 """ArticleEssence: Semantic fingerprint of academic paper for structured analysis."""
 
-from typing import List, Self
+from typing import List
 
-from fabricatio.models.generic import Display, PersistentAble, ProposedAble, Vectorizable
+from fabricatio.models.extra.rag import MilvusDataBase
+from fabricatio.models.generic import PersistentAble, SketchedAble
 from pydantic import BaseModel
 
 
@@ -54,7 +55,7 @@ class Highlightings(BaseModel):
     """
 
 
-class ArticleEssence(ProposedAble, Display, PersistentAble, Vectorizable):
+class ArticleEssence(SketchedAble, PersistentAble, MilvusDataBase):
     """Structured representation of a scientific article's core elements in its original language."""
 
     language: str
@@ -93,7 +94,7 @@ class ArticleEssence(ProposedAble, Display, PersistentAble, Vectorizable):
     bibtex_cite_key: str
     """Bibtex cite key of the original article."""
 
-    def update_cite_key(self, new_cite_key: str) -> Self:
-        """Update the bibtex_cite_key of the article."""
-        self.bibtex_cite_key = new_cite_key
-        return self
+    def _prepare_vectorization_inner(self) -> str:
+        return self.compact()
+
+

fabricatio/models/extra/article_main.py

@@ -2,6 +2,7 @@
 
 from typing import Dict, Generator, List, Self, Tuple, override
 
+from fabricatio.decorators import precheck_package
 from fabricatio.fs.readers import extract_sections
 from fabricatio.journal import logger
 from fabricatio.models.extra.article_base import (
@@ -14,7 +15,7 @@ from fabricatio.models.extra.article_outline import (
     ArticleOutline,
 )
 from fabricatio.models.generic import Described, PersistentAble, SequencePatch, SketchedAble, WithRef, WordCount
-from fabricatio.rust import word_count
+from fabricatio.rust import convert_all_block_tex, convert_all_inline_tex, word_count
 from pydantic import Field
 
 PARAGRAPH_SEP = "// - - -"
@@ -153,6 +154,26 @@ class Article(
             "Original Article": self.display(),
         }
 
+    def convert_tex(self) -> Self:
+        """Convert tex to typst code."""
+        for _, _, subsec in self.iter_subsections():
+            for p in subsec.paragraphs:
+                p.content = convert_all_inline_tex(p.content)
+                p.content = convert_all_block_tex(p.content)
+        return self
+
+    def fix_wrapper(self) -> Self:
+        """Fix wrapper."""
+        for _, _, subsec in self.iter_subsections():
+            for p in subsec.paragraphs:
+                p.content = (
+                    p.content.replace(r" \( ", "$")
+                    .replace(r" \) ", "$")
+                    .replace("\\[\n", "$$\n")
+                    .replace("\n\\]", "\n$$")
+                )
+        return self
+
     @override
     def iter_subsections(self) -> Generator[Tuple[ArticleChapter, ArticleSection, ArticleSubsection], None, None]:
         return super().iter_subsections()  # pyright: ignore [reportReturnType]
@@ -204,3 +225,20 @@ class Article(
             expected_word_count=word_count(body),
             abstract="",
         )
+
+    @classmethod
+    def from_mixed_source(cls, article_outline: ArticleOutline, typst_code: str) -> Self:
+        """Generates an article from the given outline and Typst code."""
+        self = cls.from_typst_code(article_outline.title, typst_code)
+        self.expected_word_count = article_outline.expected_word_count
+        self.description = article_outline.description
+        for a, o in zip(self.iter_dfs(), article_outline.iter_dfs(), strict=True):
+            a.update_metadata(o)
+        return self.update_ref(article_outline)
+
+    @precheck_package(
+        "questionary", "'questionary' is required to run this function. Have you installed `fabricatio[qa]`?."
+    )
+    def edit_titles(self) -> Self:
+        for a in self.iter_dfs():
+            pass

fabricatio/models/extra/problem.py

@@ -3,12 +3,12 @@
 from itertools import chain
 from typing import Any, List, Optional, Self, Tuple, Unpack
 
+from pydantic import Field
+from rich import print as r_print
+
 from fabricatio.journal import logger
 from fabricatio.models.generic import SketchedAble, WithBriefing
 from fabricatio.utils import ask_edit
-from pydantic import Field
-from questionary import Choice, checkbox, text
-from rich import print as r_print
 
 
 class Problem(SketchedAble, WithBriefing):
@@ -74,6 +74,8 @@ class ProblemSolutions(SketchedAble):
         return len(self.solutions) > 0
 
     async def edit_problem(self) -> Self:
+        from questionary import text
+
         """Interactively edit the problem description."""
         self.problem = Problem.model_validate_strings(
             await text("Please edit the problem below:", default=self.problem.display()).ask_async()
@@ -127,6 +129,8 @@ class Improvement(SketchedAble):
         Returns:
             Self: The current instance with filtered problems and solutions.
         """
+        from questionary import Choice, checkbox
+
         # Choose the problems to retain
         chosen_ones: List[ProblemSolutions] = await checkbox(
             "Please choose the problems you want to retain.(Default: retain all)",