fabricatio 0.2.9.dev3__cp312-cp312-win_amd64.whl → 0.2.10__cp312-cp312-win_amd64.whl

fabricatio/models/adv_kwargs_types.py

@@ -1,4 +1,8 @@
 """A module containing kwargs types for content correction and checking operations."""
+
+from importlib.util import find_spec
+from typing import NotRequired, TypedDict
+
 from fabricatio.models.extra.problem import Improvement
 from fabricatio.models.extra.rule import RuleSet
 from fabricatio.models.generic import SketchedAble
@@ -23,3 +27,34 @@ class CheckKwargs(ReferencedKwargs[Improvement], total=False):
     """
 
     ruleset: RuleSet
+
+
+if find_spec("pymilvus"):
+    from pymilvus import CollectionSchema
+    from pymilvus.milvus_client import IndexParams
+
+    class CollectionConfigKwargs(TypedDict, total=False):
+        """Configuration parameters for a vector collection.
+
+        These arguments are typically used when configuring connections to vector databases.
+        """
+
+        dimension: int | None
+        primary_field_name: str
+        id_type: str
+        vector_field_name: str
+        metric_type: str
+        timeout: float | None
+        schema: CollectionSchema | None
+        index_params: IndexParams | None
+
+    class FetchKwargs(TypedDict):
+        """Arguments for fetching data from vector collections.
+
+        Controls how data is retrieved from vector databases, including filtering
+        and result limiting parameters.
+        """
+
+        collection_name: NotRequired[str | None]
+        similarity_threshold: NotRequired[float]
+        result_per_query: NotRequired[int]

fabricatio/models/events.py

@@ -3,7 +3,7 @@
 from typing import List, Self, Union
 
 from fabricatio.config import configs
-from fabricatio.models.utils import TaskStatus
+from fabricatio.constants import TaskStatus
 from pydantic import BaseModel, ConfigDict, Field
 
 type EventLike = Union[str, List[str], "Event"]
@@ -77,23 +77,23 @@ class Event(BaseModel):
 
     def push_pending(self) -> Self:
         """Push a pending segment to the event."""
-        return self.push(TaskStatus.Pending.value)
+        return self.push(TaskStatus.Pending)
 
     def push_running(self) -> Self:
         """Push a running segment to the event."""
-        return self.push(TaskStatus.Running.value)
+        return self.push(TaskStatus.Running)
 
     def push_finished(self) -> Self:
         """Push a finished segment to the event."""
-        return self.push(TaskStatus.Finished.value)
+        return self.push(TaskStatus.Finished)
 
     def push_failed(self) -> Self:
         """Push a failed segment to the event."""
-        return self.push(TaskStatus.Failed.value)
+        return self.push(TaskStatus.Failed)
 
     def push_cancelled(self) -> Self:
         """Push a cancelled segment to the event."""
-        return self.push(TaskStatus.Cancelled.value)
+        return self.push(TaskStatus.Cancelled)
 
     def pop(self) -> str:
         """Pop a segment from the event."""

fabricatio/models/extra/advanced_judge.py

@@ -2,17 +2,17 @@
 
 from typing import List
 
-from fabricatio.models.generic import Display, ProposedAble
+from fabricatio.models.generic import SketchedAble
 
 
-class JudgeMent(ProposedAble, Display):
+class JudgeMent(SketchedAble):
     """Represents a judgment result containing supporting/denying evidence and final verdict.
 
     The class stores both affirmative and denies evidence, truth and reasons lists along with the final boolean judgment.
     """
 
     issue_to_judge: str
-    """The issue to be judged"""
+    """The issue to be judged, including the original question and context"""
 
     deny_evidence: List[str]
     """List of clues supporting the denial."""
@@ -21,7 +21,7 @@ class JudgeMent(ProposedAble, Display):
     """List of clues supporting the affirmation."""
 
     final_judgement: bool
-    """The final judgment made according to all extracted clues."""
+    """The final judgment made according to all extracted clues. true for the `issue_to_judge` is correct and false for incorrect."""
 
     def __bool__(self) -> bool:
         """Return the final judgment value.

fabricatio/models/extra/aricle_rag.py

@@ -0,0 +1,170 @@
+"""A Module containing the article rag models."""
+
+from pathlib import Path
+from typing import ClassVar, Dict, List, Optional, Self, Unpack
+
+from fabricatio.rust import BibManager, split_into_chunks, is_chinese
+from more_itertools.recipes import flatten
+from pydantic import Field
+
+from fabricatio.fs import safe_text_read
+from fabricatio.journal import logger
+from fabricatio.models.extra.article_main import ArticleSubsection
+from fabricatio.models.extra.rag import MilvusDataBase
+from fabricatio.models.generic import AsPrompt
+from fabricatio.models.kwargs_types import ChunkKwargs
+from fabricatio.utils import ok, wrapp_in_block
+
+
+class ArticleChunk(MilvusDataBase, AsPrompt):
+    """The chunk of an article."""
+
+    etc_word: ClassVar[str] = "等"
+    and_word: ClassVar[str] = "与"
+    _cite_number: Optional[int] = None
+
+    head_split: ClassVar[List[str]] = [
+        "引 言",
+        "引言",
+        "绪 论",
+        "绪论",
+        "前言",
+        "INTRODUCTION",
+        "Introduction",
+    ]
+    tail_split: ClassVar[List[str]] = [
+        "参 考 文 献",
+        "参  考  文  献",
+        "参考文献",
+        "REFERENCES",
+        "References",
+        "Bibliography",
+        "Reference",
+    ]
+    chunk: str
+    """The segment of the article"""
+    year: int
+    """The year of the article"""
+    authors: List[str] = Field(default_factory=list)
+    """The authors of the article"""
+    article_title: str
+    """The title of the article"""
+    bibtex_cite_key: str
+    """The bibtex cite key of the article"""
+
+    def _as_prompt_inner(self) -> Dict[str, str]:
+        return {
+            f"{ok(self._cite_number, 'You need to update cite number first.')}th reference `{self.article_title}`": f"{wrapp_in_block(self.chunk, 'Referring Content')}\n"
+                                                                                                                    f"Authors: {';'.join(self.authors)}\n"
+                                                                                                                    f"Published Year: {self.year}\n"
+        }
+
+    def _prepare_vectorization_inner(self) -> str:
+        return self.chunk
+
+    @classmethod
+    def from_file[P: str | Path](
+            cls, path: P | List[P], bib_mgr: BibManager, **kwargs: Unpack[ChunkKwargs]
+    ) -> List[Self]:
+        """Load the article chunks from the file."""
+        if isinstance(path, list):
+            result = list(flatten(cls._from_file_inner(p, bib_mgr, **kwargs) for p in path))
+            logger.debug(f"Number of chunks created from list of files: {len(result)}")
+            return result
+
+        return cls._from_file_inner(path, bib_mgr, **kwargs)
+
+    @classmethod
+    def _from_file_inner(cls, path: str | Path, bib_mgr: BibManager, **kwargs: Unpack[ChunkKwargs]) -> List[Self]:
+        path = Path(path)
+
+        title_seg = path.stem.split(" - ").pop()
+
+        key = (
+                bib_mgr.get_cite_key_by_title(title_seg)
+                or bib_mgr.get_cite_key_by_title_fuzzy(title_seg)
+                or bib_mgr.get_cite_key_fuzzy(path.stem)
+        )
+        if key is None:
+            logger.warning(f"no cite key found for {path.as_posix()}, skip.")
+            return []
+        authors = ok(bib_mgr.get_author_by_key(key), f"no author found for {key}")
+        year = ok(bib_mgr.get_year_by_key(key), f"no year found for {key}")
+        article_title = ok(bib_mgr.get_title_by_key(key), f"no title found for {key}")
+
+        result = [
+            cls(chunk=c, year=year, authors=authors, article_title=article_title, bibtex_cite_key=key)
+            for c in split_into_chunks(cls.strip(safe_text_read(path)), **kwargs)
+        ]
+        logger.debug(f"Number of chunks created from file {path.as_posix()}: {len(result)}")
+        return result
+
+    @classmethod
+    def strip(cls, string: str) -> str:
+        """Strip the head and tail of the string."""
+        logger.debug(f"String length before strip: {(original := len(string))}")
+        for split in (s for s in cls.head_split if s in string):
+            logger.debug(f"Strip head using {split}")
+            parts = string.split(split)
+            string = split.join(parts[1:]) if len(parts) > 1 else parts[0]
+            break
+        logger.debug(
+            f"String length after head strip: {(stripped_len := len(string))}, decreased by {(d := original - stripped_len)}"
+        )
+        if not d:
+            logger.warning("No decrease at head strip, which is might be abnormal.")
+        for split in (s for s in cls.tail_split if s in string):
+            logger.debug(f"Strip tail using {split}")
+            parts = string.split(split)
+            string = split.join(parts[:-1]) if len(parts) > 1 else parts[0]
+            break
+        logger.debug(f"String length after tail strip: {len(string)}, decreased by {(d := stripped_len - len(string))}")
+        if not d:
+            logger.warning("No decrease at tail strip, which is might be abnormal.")
+
+        return string
+
+    def as_typst_cite(self) -> str:
+        """As typst cite."""
+        return f"#cite(<{self.bibtex_cite_key}>)"
+
+    @property
+    def auther_firstnames(self) -> List[str]:
+        """Get the first name of the authors."""
+        ret = []
+        for n in self.authors:
+            if is_chinese(n):
+                ret.append(n[0])
+            else:
+                ret.append(n.split()[-1])
+        return ret
+
+    def as_auther_seq(self) -> str:
+        """Get the auther sequence."""
+        match len(self.authors):
+            case 0:
+                raise ValueError("No authors found")
+            case 1:
+                return f"（{self.auther_firstnames[0]}，{self.year}）{self.as_typst_cite()}"
+            case 2:
+                return f"（{self.auther_firstnames[0]}{self.and_word}{self.auther_firstnames[1]}，{self.year}）{self.as_typst_cite()}"
+            case 3:
+                return f"（{self.auther_firstnames[0]}，{self.auther_firstnames[1]}{self.and_word}{self.auther_firstnames[2]}，{self.year}）{self.as_typst_cite()}"
+            case _:
+                return f"（{self.auther_firstnames[0]}，{self.auther_firstnames[1]}{self.and_word}{self.auther_firstnames[2]}{self.etc_word}，{self.year}）{self.as_typst_cite()}"
+
+    def update_cite_number(self, cite_number: int) -> Self:
+        """Update the cite number."""
+        self._cite_number = cite_number
+        return self
+
+    def replace_cite(self, string: str, left_char: str = "[[", right_char: str = "]]") -> str:
+        """Replace the cite number in the string."""
+        return string.replace(f"{left_char}{ok(self._cite_number)}{right_char}", self.as_auther_seq())
+
+    def apply(self, article_subsection: ArticleSubsection) -> ArticleSubsection:
+        """Apply the patch to the article subsection."""
+        for p in article_subsection.paragraphs:
+            p.content = self.replace_cite(p.content)
+
+        return article_subsection

fabricatio/models/extra/article_base.py

@@ -2,8 +2,7 @@
 
 from abc import ABC, abstractmethod
 from enum import StrEnum
-from itertools import chain
-from typing import Generator, List, Optional, Self, Tuple, overload
+from typing import Generator, List, Optional, Self, Tuple
 
 from fabricatio.models.generic import (
     AsPrompt,
@@ -15,9 +14,11 @@ from fabricatio.models.generic import (
     PersistentAble,
     ProposedUpdateAble,
     ResolveUpdateConflict,
-    SequencePatch,
     SketchedAble,
+    Titled,
+    WordCount,
 )
+from pydantic import Field
 
 
 class ReferringType(StrEnum):
@@ -28,102 +29,25 @@ class ReferringType(StrEnum):
     SUBSECTION = "subsection"
 
 
-type RefKey = Tuple[str, Optional[str], Optional[str]]
-
-
-class ArticleRef(ProposedUpdateAble):
-    """Reference to a specific chapter, section or subsection within the article. You SHALL not refer to an article component that is external and not present within our own article.
-
-    Examples:
-        - Referring to a chapter titled `Introduction`:
-            Using Python
-            ```python
-            ArticleRef(referred_chapter_title="Introduction")
-            ```
-            Using JSON
-            ```json
-            {referred_chapter_title="Introduction"}
-            ```
-        - Referring to a section titled `Background` under the `Introduction` chapter:
-            Using Python
-            ```python
-            ArticleRef(referred_chapter_title="Introduction", referred_section_title="Background")
-            ```
-            Using JSON
-            ```json
-            {referred_chapter_title="Introduction", referred_section_title="Background"}
-            ```
-        - Referring to a subsection titled `Related Work` under the `Background` section of the `Introduction` chapter:
-            Using Python
-            ```python
-            ArticleRef(referred_chapter_title="Introduction", referred_section_title="Background", referred_subsection_title="Related Work")
-            ```
-            Using JSON
-            ```json
-            {referred_chapter_title="Introduction", referred_section_title="Background", referred_subsection_title="Related Work"}
-            ```
-    """
-
-    referred_chapter_title: str
-    """`title` Field of the referenced chapter"""
-    referred_section_title: Optional[str] = None
-    """`title` Field of the referenced section."""
-    referred_subsection_title: Optional[str] = None
-    """`title` Field of the referenced subsection."""
-
-    def update_from_inner(self, other: Self) -> Self:
-        """Updates the current instance with the attributes of another instance."""
-        self.referred_chapter_title = other.referred_chapter_title
-        self.referred_section_title = other.referred_section_title
-        self.referred_subsection_title = other.referred_subsection_title
-        return self
-
-    def deref(self, article: "ArticleBase") -> Optional["ArticleOutlineBase"]:
-        """Dereference the reference to the actual section or subsection within the provided article.
-
-        Args:
-            article (ArticleOutline | Article): The article to dereference the reference from.
 
-        Returns:
-            ArticleMainBase | ArticleOutline | None: The dereferenced section or subsection, or None if not found.
-        """
-        chap = next((chap for chap in article.chapters if chap.title == self.referred_chapter_title), None)
-        if self.referred_section_title is None or chap is None:
-            return chap
-        sec = next((sec for sec in chap.sections if sec.title == self.referred_section_title), None)
-        if self.referred_subsection_title is None or sec is None:
-            return sec
-        return next((subsec for subsec in sec.subsections if subsec.title == self.referred_subsection_title), None)
+type RefKey = Tuple[str, Optional[str], Optional[str]]
 
-    @property
-    def referring_type(self) -> ReferringType:
-        """Determine the type of reference based on the presence of specific attributes."""
-        if self.referred_subsection_title is not None:
-            return ReferringType.SUBSECTION
-        if self.referred_section_title is not None:
-            return ReferringType.SECTION
-        return ReferringType.CHAPTER
 
 
-class ArticleMetaData(SketchedAble, Described, Language):
+class ArticleMetaData(SketchedAble, Described, WordCount, Titled, Language):
     """Metadata for an article component."""
 
-    support_to: List[ArticleRef]
-    """List of references to other component of this articles that this component supports."""
-    depend_on: List[ArticleRef]
-    """List of references to other component of this articles that this component depends on."""
+    description: str = Field(
+        alias="elaboration",
+        description=Described.model_fields["description"].description,
+    )
 
-    writing_aim: List[str]
-    """List of writing aims of the research component in academic style."""
-    title: str
-    """Do not add any prefix or suffix to the title. should not contain special characters."""
+    title: str = Field(alias="heading", description=Titled.model_fields["title"].description)
 
-    expected_word_count: int
-    """Expected word count of this research component."""
+    aims: List[str]
+    """List of writing aims of the research component in academic style."""
 
 
-class ArticleRefSequencePatch(SequencePatch[ArticleRef]):
-    """Patch for article refs."""
 
 
 class ArticleOutlineBase(
@@ -143,12 +67,8 @@ class ArticleOutlineBase(
 
     def update_metadata(self, other: ArticleMetaData) -> Self:
         """Updates the metadata of the current instance with the attributes of another instance."""
-        self.support_to.clear()
-        self.support_to.extend(other.support_to)
-        self.depend_on.clear()
-        self.depend_on.extend(other.depend_on)
-        self.writing_aim.clear()
-        self.writing_aim.extend(other.writing_aim)
+        self.aims.clear()
+        self.aims.extend(other.aims)
         self.description = other.description
         return self
 
@@ -272,22 +192,19 @@ class ChapterBase[T: SectionBase](ArticleOutlineBase):
         return ""
 
 
-class ArticleBase[T: ChapterBase](FinalizedDumpAble, AsPrompt, Language, ABC):
+class ArticleBase[T: ChapterBase](FinalizedDumpAble, AsPrompt, WordCount, Described, Titled, Language, ABC):
     """Base class for article outlines."""
 
-    title: str
-    """Title of the academic paper."""
-
-    prospect: str
-    """Consolidated research statement with four pillars:
-    1. Problem Identification: Current limitations
-    2. Methodological Response: Technical approach
-    3. Empirical Validation: Evaluation strategy
-    4. Scholarly Impact: Field contributions
-    """
+    title: str = Field(alias="heading", description=Titled.model_fields["title"].description)
+    description: str = Field(alias="abstract")
+    """The abstract serves as a concise summary of an academic article, encapsulating its core purpose, methodologies, key results,
+    and conclusions while enabling readers to rapidly assess the relevance and significance of the study.
+    Functioning as the article's distilled essence, it succinctly articulates the research problem, objectives,
+    and scope, providing a roadmap for the full text while also facilitating database indexing, literature reviews,
+    and citation tracking through standardized metadata. Additionally, it acts as an accessibility gateway,
+    allowing scholars to gauge the study's contribution to existing knowledge, its methodological rigor,
+    and its broader implications without engaging with the entire manuscript, thereby optimizing scholarly communication efficiency."""
 
-    abstract: str
-    """The abstract is a concise summary of the academic paper's main findings."""
     chapters: List[T]
     """Chapters of the article. Contains at least one chapter. You can also add more as needed."""
 
@@ -317,34 +234,6 @@ class ArticleBase[T: ChapterBase](FinalizedDumpAble, AsPrompt, Language, ABC):
                 yield sec
                 yield from sec.subsections
 
-    def iter_support_on(self, rev: bool = False) -> Generator[ArticleRef, None, None]:
-        """Iterates over all references that the article components support.
-
-        Args:
-            rev (bool): If True, iterate in reverse order.
-
-        Yields:
-            ArticleRef: Each reference that the article components support.
-        """
-        if rev:
-            yield from chain(*[a.support_to for a in self.iter_dfs_rev()])
-            return
-        yield from chain(*[a.support_to for a in self.iter_dfs()])
-
-    def iter_depend_on(self, rev: bool = False) -> Generator[ArticleRef, None, None]:
-        """Iterates over all references that the article components depend on.
-
-        Args:
-            rev (bool): If True, iterate in reverse order.
-
-        Yields:
-            ArticleRef: Each reference that the article components depend on.
-        """
-        if rev:
-            yield from chain(*[a.depend_on for a in self.iter_dfs_rev()])
-            return
-        yield from chain(*[a.depend_on for a in self.iter_dfs()])
-
     def iter_sections(self) -> Generator[Tuple[ChapterBase, SectionBase], None, None]:
         """Iterates through all sections in the article.
 
@@ -378,12 +267,6 @@ class ArticleBase[T: ChapterBase](FinalizedDumpAble, AsPrompt, Language, ABC):
         """Gathers all introspected components in the article structure."""
         return "\n".join([i for component in self.chapters if (i := component.introspect())])
 
-    @overload
-    def find_illegal_ref(self, gather_identical: bool) -> Optional[Tuple[ArticleRef | List[ArticleRef], str]]: ...
-
-    @overload
-    def find_illegal_ref(self) -> Optional[Tuple[ArticleRef, str]]: ...
-
     def iter_chap_title(self) -> Generator[str, None, None]:
         """Iterates through all chapter titles in the article."""
         for chap in self.chapters:
@@ -399,75 +282,6 @@ class ArticleBase[T: ChapterBase](FinalizedDumpAble, AsPrompt, Language, ABC):
         for _, _, subsec in self.iter_subsections():
             yield subsec.title
 
-    def find_illegal_ref(self, gather_identical: bool = False) -> Optional[Tuple[ArticleRef | List[ArticleRef], str]]:
-        """Finds the first illegal component in the outline.
-
-        Returns:
-            Tuple[ArticleOutlineBase, str]: A tuple containing the illegal component and an error message.
-        """
-        summary = ""
-        chap_titles_set = set(self.iter_chap_title())
-        sec_titles_set = set(self.iter_section_title())
-        subsec_titles_set = set(self.iter_subsection_title())
-
-        for component in self.iter_dfs_rev():
-            for ref in chain(component.depend_on, component.support_to):
-                if not ref.deref(self):
-                    summary += f"Invalid internal reference in `{component.__class__.__name__}` titled `{component.title}`, because the referred {ref.referring_type} is not exists within the article, see the original obj dump: {ref.model_dump()}\n"
-
-                    if ref.referred_chapter_title not in (chap_titles_set):
-                        summary += f"Chapter titled `{ref.referred_chapter_title}` is not any of {chap_titles_set}\n"
-                    if ref.referred_section_title and ref.referred_section_title not in (sec_titles_set):
-                        summary += f"Section Titled `{ref.referred_section_title}` is not any of {sec_titles_set}\n"
-                    if ref.referred_subsection_title and ref.referred_subsection_title not in (subsec_titles_set):
-                        summary += (
-                            f"Subsection Titled `{ref.referred_subsection_title}` is not any of {subsec_titles_set}"
-                        )
-
-                if summary:
-                    return (
-                        (
-                            [
-                                identical_ref
-                                for identical_ref in chain(self.iter_depend_on(), self.iter_support_on())
-                                if identical_ref == ref
-                            ],
-                            summary,
-                        )
-                        if gather_identical
-                        else (ref, summary)
-                    )
-
-        return None
-
-    def gather_illegal_ref(self) -> Tuple[List[ArticleRef], str]:
-        """Gathers all illegal references in the article."""
-        summary = []
-        chap_titles_set = set(self.iter_chap_title())
-        sec_titles_set = set(self.iter_section_title())
-        subsec_titles_set = set(self.iter_subsection_title())
-        res_seq = []
-
-        for component in self.iter_dfs():
-            for ref in (
-                r for r in chain(component.depend_on, component.support_to) if not r.deref(self) and r not in res_seq
-            ):
-                res_seq.append(ref)
-                if ref.referred_chapter_title not in chap_titles_set:
-                    summary.append(
-                        f"Chapter titled `{ref.referred_chapter_title}` is not exist, since it is not any of {chap_titles_set}."
-                    )
-                if ref.referred_section_title and (ref.referred_section_title not in sec_titles_set):
-                    summary.append(
-                        f"Section Titled `{ref.referred_section_title}` is not exist, since it is not any of {sec_titles_set}"
-                    )
-                if ref.referred_subsection_title and (ref.referred_subsection_title not in subsec_titles_set):
-                    summary.append(
-                        f"Subsection Titled `{ref.referred_subsection_title}` is not exist, since it is not any of {subsec_titles_set}"
-                    )
-
-        return res_seq, "\n".join(summary)
-
     def finalized_dump(self) -> str:
         """Generates standardized hierarchical markup for academic publishing systems.
 

fabricatio/models/extra/article_essence.py

@@ -1,8 +1,9 @@
 """ArticleEssence: Semantic fingerprint of academic paper for structured analysis."""
 
-from typing import List, Self
+from typing import List
 
-from fabricatio.models.generic import Display, PersistentAble, ProposedAble, Vectorizable
+from fabricatio.models.extra.rag import MilvusDataBase
+from fabricatio.models.generic import PersistentAble, SketchedAble
 from pydantic import BaseModel
 
 
@@ -54,7 +55,7 @@ class Highlightings(BaseModel):
     """
 
 
-class ArticleEssence(ProposedAble, Display, PersistentAble, Vectorizable):
+class ArticleEssence(SketchedAble, PersistentAble, MilvusDataBase):
     """Structured representation of a scientific article's core elements in its original language."""
 
     language: str
@@ -93,7 +94,7 @@ class ArticleEssence(ProposedAble, Display, PersistentAble, Vectorizable):
     bibtex_cite_key: str
     """Bibtex cite key of the original article."""
 
-    def update_cite_key(self, new_cite_key: str) -> Self:
-        """Update the bibtex_cite_key of the article."""
-        self.bibtex_cite_key = new_cite_key
-        return self
+    def _prepare_vectorization_inner(self) -> str:
+        return self.compact()
+
+