fabricatio 0.2.6.dev3__cp39-cp39-win_amd64.whl

fabricatio/models/utils.py

@@ -0,0 +1,167 @@
+"""A module containing utility classes for the models."""
+
+from enum import Enum
+from typing import Any, Dict, List, Literal, Optional, Self
+
+from pydantic import BaseModel, ConfigDict, Field
+from questionary import text
+
+
+class Message(BaseModel):
+    """A class representing a message."""
+
+    model_config = ConfigDict(use_attribute_docstrings=True)
+    role: Literal["user", "system", "assistant"] = Field(default="user")
+    """
+    Who is sending the message.
+    """
+    content: str = Field(default="")
+    """
+    The content of the message.
+    """
+
+
+class Messages(list):
+    """A list of messages."""
+
+    def add_message(self, role: Literal["user", "system", "assistant"], content: str) -> Self:
+        """Adds a message to the list with the specified role and content.
+
+        Args:
+            role (Literal["user", "system", "assistant"]): The role of the message sender.
+            content (str): The content of the message.
+
+        Returns:
+            Self: The current instance of Messages to allow method chaining.
+        """
+        if content:
+            self.append(Message(role=role, content=content))
+        return self
+
+    def add_user_message(self, content: str) -> Self:
+        """Adds a user message to the list with the specified content.
+
+        Args:
+            content (str): The content of the user message.
+
+        Returns:
+            Self: The current instance of Messages to allow method chaining.
+        """
+        return self.add_message("user", content)
+
+    def add_system_message(self, content: str) -> Self:
+        """Adds a system message to the list with the specified content.
+
+        Args:
+            content (str): The content of the system message.
+
+        Returns:
+            Self: The current instance of Messages to allow method chaining.
+        """
+        return self.add_message("system", content)
+
+    def add_assistant_message(self, content: str) -> Self:
+        """Adds an assistant message to the list with the specified content.
+
+        Args:
+            content (str): The content of the assistant message.
+
+        Returns:
+            Self: The current instance of Messages to allow method chaining.
+        """
+        return self.add_message("assistant", content)
+
+    def as_list(self) -> List[Dict[str, str]]:
+        """Converts the messages to a list of dictionaries.
+
+        Returns:
+            list[dict]: A list of dictionaries representing the messages.
+        """
+        return [message.model_dump() for message in self]
+
+
+class MilvusData(BaseModel):
+    """A class representing data stored in Milvus."""
+
+    model_config = ConfigDict(use_attribute_docstrings=True)
+    id: Optional[int] = Field(default=None)
+    """The identifier of the data."""
+
+    vector: List[float]
+    """The vector representation of the data."""
+
+    text: str
+    """The text representation of the data."""
+
+    subject: Optional[str] = Field(default=None)
+    """A subject label that we use to demo metadata filtering later."""
+
+    def prepare_insertion(self) -> Dict[str, Any]:
+        """Prepares the data for insertion into Milvus.
+
+        Returns:
+            dict: A dictionary containing the data to be inserted into Milvus.
+        """
+        return self.model_dump(exclude_none=True)
+
+    def update_subject(self, new_subject: str) -> Self:
+        """Updates the subject label of the data.
+
+        Args:
+            new_subject (str): The new subject label.
+
+        Returns:
+            Self: The updated instance of MilvusData.
+        """
+        self.subject = new_subject
+        return self
+
+    def update_id(self, new_id: int) -> Self:
+        """Updates the identifier of the data.
+
+        Args:
+            new_id (int): The new identifier.
+
+        Returns:
+            Self: The updated instance of MilvusData.
+        """
+        self.id = new_id
+        return self
+
+
+class TaskStatus(Enum):
+    """An enumeration representing the status of a task.
+
+    Attributes:
+        Pending: The task is pending.
+        Running: The task is currently running.
+        Finished: The task has been successfully completed.
+        Failed: The task has failed.
+        Cancelled: The task has been cancelled.
+    """
+
+    Pending = "pending"
+    Running = "running"
+    Finished = "finished"
+    Failed = "failed"
+    Cancelled = "cancelled"
+
+
+async def ask_edit(
+    text_seq: List[str],
+) -> List[str]:
+    """Asks the user to edit a list of texts.
+
+    Args:
+        text_seq (List[str]): A list of texts to be edited.
+
+    Returns:
+        List[str]: A list of edited texts.
+        If the user does not edit a text, it will not be included in the returned list.
+    """
+    res = []
+    for i, t in enumerate(text_seq):
+        edited = await text(f"[{i}] ", default=t).ask_async()
+        if edited:
+            res.append(edited)
+    return res

fabricatio/parser.py

@@ -0,0 +1,149 @@
+"""A module to parse text using regular expressions."""
+
+from typing import Any, Callable, Iterable, List, Optional, Self, Tuple, Type
+
+import orjson
+import regex
+from json_repair import repair_json
+from pydantic import BaseModel, ConfigDict, Field, PositiveInt, PrivateAttr, ValidationError
+from regex import Pattern, compile
+
+from fabricatio.config import configs
+from fabricatio.journal import logger
+
+
+class Capture(BaseModel):
+    """A class to capture patterns in text using regular expressions.
+
+    Attributes:
+        pattern (str): The regular expression pattern to search for.
+        _compiled (Pattern): The compiled regular expression pattern.
+    """
+
+    model_config = ConfigDict(use_attribute_docstrings=True)
+    target_groups: Tuple[int, ...] = Field(default_factory=tuple)
+    """The target groups to capture from the pattern."""
+    pattern: str = Field(frozen=True)
+    """The regular expression pattern to search for."""
+    flags: PositiveInt = Field(default=regex.DOTALL | regex.MULTILINE | regex.IGNORECASE, frozen=True)
+    """The flags to use when compiling the regular expression pattern."""
+    capture_type: Optional[str] = None
+    """The type of capture to perform, e.g., 'json', which is used to dispatch the fixer accordingly."""
+    _compiled: Pattern = PrivateAttr()
+
+    def model_post_init(self, __context: Any) -> None:
+        """Initialize the compiled pattern."""
+        self._compiled = compile(self.pattern, self.flags)
+
+    def fix[T](self, text: str | Iterable[str] | T) -> str | List[str] | T:
+        """Fix the text using the pattern.
+
+        Args:
+            text (str | List[str]): The text to fix.
+
+        Returns:
+            str | List[str]: The fixed text with the same type as input.
+        """
+        match self.capture_type:
+            case "json":
+                if isinstance(text, str):
+                    return repair_json(text, ensure_ascii=False)
+                return [repair_json(item, ensure_ascii=False) for item in text]
+            case _:
+                return text
+
+    def capture(self, text: str) -> Tuple[str, ...] | str | None:
+        """Capture the first occurrence of the pattern in the given text.
+
+        Args:
+            text (str): The text to search the pattern in.
+
+        Returns:
+            str | None: The captured text if the pattern is found, otherwise None.
+
+        """
+        match = self._compiled.search(text)
+        if match is None:
+            return None
+        groups = self.fix(match.groups()) if configs.general.use_json_repair else match.groups()
+        if self.target_groups:
+            cap = tuple(groups[g - 1] for g in self.target_groups)
+            logger.debug(f"Captured text: {'\n\n'.join(cap)}")
+            return cap
+        cap = groups[0]
+        logger.debug(f"Captured text: \n{cap}")
+        return cap
+
+    def convert_with[T](self, text: str, convertor: Callable[[Tuple[str, ...]], T] | Callable[[str], T]) -> T | None:
+        """Convert the given text using the pattern.
+
+        Args:
+            text (str): The text to search the pattern in.
+            convertor (Callable[[Tuple[str, ...]], T] | Callable[[str], T]): The function to convert the captured text.
+
+        Returns:
+            str | None: The converted text if the pattern is found, otherwise None.
+        """
+        if (cap := self.capture(text)) is None:
+            return None
+        try:
+            return convertor(cap)
+        except (ValueError, SyntaxError, ValidationError) as e:
+            logger.error(f"Failed to convert text using {convertor.__name__} to convert.\nerror: {e}\n {cap}")
+            return None
+
+    def validate_with[K, T, E](
+        self,
+        text: str,
+        target_type: Type[T],
+        elements_type: Optional[Type[E]] = None,
+        length: Optional[int] = None,
+        deserializer: Callable[[Tuple[str, ...]], K] | Callable[[str], K] = orjson.loads,
+    ) -> T | None:
+        """Validate the given text using the pattern.
+
+        Args:
+            text (str): The text to search the pattern in.
+            target_type (Type[T]): The expected type of the output, dict or list.
+            elements_type (Optional[Type[E]]): The expected type of the elements in the output dict keys or list elements.
+            length (Optional[int]): The expected length of the output, bool(length)==False means no length validation.
+            deserializer (Callable[[Tuple[str, ...]], K] | Callable[[str], K]): The function to deserialize the captured text.
+
+        Returns:
+            T | None: The validated text if the pattern is found and the output is of the expected type, otherwise None.
+        """
+        judges = [lambda output_obj: isinstance(output_obj, target_type)]
+        if elements_type:
+            judges.append(lambda output_obj: all(isinstance(e, elements_type) for e in output_obj))
+        if length:
+            judges.append(lambda output_obj: len(output_obj) == length)
+
+        if (out := self.convert_with(text, deserializer)) and all(j(out) for j in judges):
+            return out
+        return None
+
+    @classmethod
+    def capture_code_block(cls, language: str) -> Self:
+        """Capture the first occurrence of a code block in the given text.
+
+        Args:
+            language (str): The text containing the code block.
+
+        Returns:
+            Self: The instance of the class with the captured code block.
+        """
+        return cls(pattern=f"```{language}\n(.*?)\n```", capture_type=language)
+
+    @classmethod
+    def capture_generic_block(cls, language: str) -> Self:
+        """Capture the first occurrence of a generic code block in the given text.
+
+        Returns:
+            Self: The instance of the class with the captured code block.
+        """
+        return cls(pattern=f"--- Start of {language} ---\n(.*?)\n--- end of {language} ---", capture_type=language)
+
+
+JsonCapture = Capture.capture_code_block("json")
+PythonCapture = Capture.capture_code_block("python")
+GenericCapture = Capture.capture_generic_block("String")

fabricatio/toolboxes/__init__.py

@@ -0,0 +1,15 @@
+"""Contains the built-in toolboxes for the Fabricatio package."""
+
+from typing import Set
+
+from fabricatio.models.tool import ToolBox
+from fabricatio.toolboxes.arithmetic import arithmetic_toolbox
+from fabricatio.toolboxes.fs import fs_toolbox
+
+basic_toolboxes: Set[ToolBox] = {arithmetic_toolbox}
+
+__all__ = [
+    "arithmetic_toolbox",
+    "basic_toolboxes",
+    "fs_toolbox",
+]

fabricatio/toolboxes/arithmetic.py

@@ -0,0 +1,62 @@
+"""Arithmetic tools for Fabricatio."""
+
+from fabricatio.models.tool import ToolBox
+
+arithmetic_toolbox = ToolBox(name="ArithmeticToolBox", description="A toolbox for arithmetic operations.")
+
+
+@arithmetic_toolbox.collect_tool
+def add(a: float, b: float) -> float:
+    """Add two numbers.
+
+    Args:
+        a (float): The first number.
+        b (float): The second number.
+
+    Returns:
+        float: The sum of the two numbers.
+    """
+    return a + b
+
+
+@arithmetic_toolbox.collect_tool
+def subtract(a: float, b: float) -> float:
+    """Subtract two numbers.
+
+    Args:
+        a (float): The first number.
+        b (float): The second number.
+
+    Returns:
+        float: The result of subtracting b from a.
+    """
+    return a - b
+
+
+@arithmetic_toolbox.collect_tool
+def multiply(a: float, b: float) -> float:
+    """Multiply two numbers.
+
+    Args:
+        a (float): The first number.
+        b (float): The second number.
+
+    Returns:
+        float: The product of the two numbers.
+    """
+    return a * b
+
+
+@arithmetic_toolbox.collect_tool
+def divide(a: float, b: float) -> float:
+    """Divide two numbers.
+
+    Args:
+        a (float): The numerator.
+        b (float): The denominator (must not be zero).
+
+    Returns:
+        float: The result of dividing a by b.
+
+    """
+    return a / b

fabricatio/toolboxes/fs.py

@@ -0,0 +1,31 @@
+"""File system tool box."""
+
+from fabricatio.fs import (
+    absolute_path,
+    copy_file,
+    create_directory,
+    delete_directory,
+    delete_file,
+    dump_text,
+    gather_files,
+    move_file,
+    safe_json_read,
+    safe_text_read,
+    tree,
+)
+from fabricatio.models.tool import ToolBox
+
+fs_toolbox = (
+    ToolBox(name="FsToolBox", description="A toolbox for basic file system operations.")
+    .add_tool(dump_text)
+    .add_tool(copy_file)
+    .add_tool(move_file)
+    .add_tool(delete_file)
+    .add_tool(tree)
+    .add_tool(delete_directory)
+    .add_tool(create_directory)
+    .add_tool(absolute_path)
+    .add_tool(safe_text_read)
+    .add_tool(safe_json_read)
+    .add_tool(gather_files)
+)

fabricatio/workflows/articles.py

@@ -0,0 +1,15 @@
+"""Store article essence in the database."""
+
+from fabricatio.actions.article import GenerateArticleProposal, GenerateOutline
+from fabricatio.actions.output import DumpFinalizedOutput
+from fabricatio.models.action import WorkFlow
+
+WriteOutlineWorkFlow = WorkFlow(
+    name="Generate Article Outline",
+    description="Generate an outline for an article. dump the outline to the given path. in typst format.",
+    steps=(
+        GenerateArticleProposal,
+        GenerateOutline(output_key="to_dump"),
+        DumpFinalizedOutput(output_key="task_output"),
+    ),
+)

fabricatio/workflows/rag.py

@@ -0,0 +1,11 @@
+"""The workflow for extracting the essence of an article and storing it in the database."""
+
+from fabricatio.actions.article import ExtractArticleEssence
+from fabricatio.actions.rag import InjectToDB
+from fabricatio.models.action import WorkFlow
+
+StoreArticle = WorkFlow(
+    name="Extract Article Essence",
+    description="Extract the essence of an article in the given path, and store it in the database.",
+    steps=(ExtractArticleEssence(output_key="to_inject"), InjectToDB(output_key="task_output")),
+)