fabricatio 0.2.9.dev3__cp312-cp312-win_amd64.whl → 0.2.10__cp312-cp312-win_amd64.whl

fabricatio/models/generic.py

@@ -3,15 +3,14 @@
 from abc import ABC, abstractmethod
 from datetime import datetime
 from pathlib import Path
-from typing import Any, Callable, Dict, Iterable, List, Optional, Self, Type, Union, final, overload
+from typing import Any, Callable, Dict, Iterable, List, Mapping, Optional, Self, Type, Union, final, overload
 
 import orjson
-import rtoml
 from fabricatio.config import configs
 from fabricatio.fs.readers import MAGIKA, safe_text_read
 from fabricatio.journal import logger
 from fabricatio.parser import JsonCapture
-from fabricatio.rust import blake3_hash
+from fabricatio.rust import blake3_hash, detect_language
 from fabricatio.rust_instances import TEMPLATE_MANAGER
 from fabricatio.utils import ok
 from litellm.utils import token_counter
@@ -36,6 +35,7 @@ class Base(BaseModel):
     The `model_config` uses `use_attribute_docstrings=True` to ensure field descriptions are
     pulled from the attribute's docstring instead of the default Pydantic behavior.
     """
+
     model_config = ConfigDict(use_attribute_docstrings=True)
 
 
@@ -45,13 +45,14 @@ class Display(Base):
     Provides methods to generate both pretty-printed and compact JSON representations of the model.
     Used for debugging and logging purposes.
     """
+
     def display(self) -> str:
         """Generate pretty-printed JSON representation.
 
         Returns:
             str: JSON string with 1-level indentation for readability
         """
-        return self.model_dump_json(indent=1)
+        return self.model_dump_json(indent=1, by_alias=True)
 
     def compact(self) -> str:
         """Generate compact JSON representation.
@@ -59,7 +60,7 @@ class Display(Base):
         Returns:
             str: Minified JSON string without whitespace
         """
-        return self.model_dump_json()
+        return self.model_dump_json(by_alias=True)
 
     @staticmethod
     def seq_display(seq: Iterable["Display"], compact: bool = False) -> str:
@@ -102,6 +103,29 @@ class Described(Base):
     this object's intent and application."""
 
 
+class Titled(Base):
+    """Class that includes a title attribute."""
+
+    title: str
+    """The title of this object, make it professional and concise.No prefixed heading number should be included."""
+
+
+class WordCount(Base):
+    """Class that includes a word count attribute."""
+
+    expected_word_count: int
+    """Expected word count of this research component."""
+
+
+class FromMapping(Base):
+    """Class that provides a method to generate a list of objects from a mapping."""
+
+    @classmethod
+    @abstractmethod
+    def from_mapping(cls, mapping: Mapping[str, Any], **kwargs: Any) -> List[Self]:
+        """Generate a list of objects from a mapping."""
+
+
 class AsPrompt(Base):
     """Class that provides a method to generate a prompt from the model.
 
@@ -194,6 +218,7 @@ class PersistentAble(Base):
     Enables saving model instances to disk with timestamped filenames and loading from persisted files.
     Implements basic versioning through filename hashing and timestamping.
     """
+
     def persist(self, path: str | Path) -> Self:
         """Save model instance to disk with versioned filename.
 
@@ -208,7 +233,7 @@ class PersistentAble(Base):
             - Hash generated from JSON content ensures uniqueness
         """
         p = Path(path)
-        out = self.model_dump_json()
+        out = self.model_dump_json(indent=1, by_alias=True)
 
         # Generate a timestamp in the format YYYYMMDD_HHMMSS
         timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
@@ -281,9 +306,17 @@ class PersistentAble(Base):
 class Language(Base):
     """Class that provides a language attribute."""
 
-    language: str
-    """The written language of this object, which should be aligned to the original requirement
-    For example if the requirement is in Chinese, the language should be set to `zh`, if the requirement is in English, the language should be set to `en` and etc."""
+    @property
+    def language(self) -> str:
+        """Get the language of the object."""
+        if isinstance(self, Described):
+            return detect_language(self.description)
+        if isinstance(self, Titled):
+            return detect_language(self.title)
+        if isinstance(self, Named):
+            return detect_language(self.name)
+
+        return detect_language(self.model_dump_json(by_alias=True))
 
 
 class ModelHash(Base):
@@ -527,7 +560,7 @@ class FinalizedDumpAble(Base):
         Returns:
             str: The finalized dump of the object.
         """
-        return self.model_dump_json()
+        return self.model_dump_json(indent=1, by_alias=True)
 
     def finalized_dump_to(self, path: str | Path) -> Self:
         """Finalize the dump of the object to a file.
@@ -639,8 +672,9 @@ class Vectorizable(Base):
     This class includes methods to prepare the model for vectorization, ensuring it fits within a specified token length.
     """
 
+    @abstractmethod
     def _prepare_vectorization_inner(self) -> str:
-        return rtoml.dumps(self.model_dump())
+        """Prepare the model for vectorization."""
 
     @final
     def prepare_vectorization(self, max_length: Optional[int] = None) -> str:
@@ -658,8 +692,7 @@ class Vectorizable(Base):
         max_length = max_length or configs.embedding.max_sequence_length
         chunk = self._prepare_vectorization_inner()
         if max_length and (length := token_counter(text=chunk)) > max_length:
-            logger.error(err := f"Chunk exceeds maximum sequence length {max_length}, got {length}, see {chunk}")
-            raise ValueError(err)
+            raise ValueError(f"Chunk exceeds maximum sequence length {max_length}, got {length}, see \n{chunk}")
 
         return chunk
 
@@ -670,6 +703,7 @@ class ScopedConfig(Base):
     Manages LLM, embedding, and vector database configurations with fallback logic.
     Allows configuration values to be overridden in a hierarchical manner.
     """
+
     llm_api_endpoint: Optional[HttpUrl] = None
     """The OpenAI API endpoint."""
 
@@ -709,6 +743,12 @@ class ScopedConfig(Base):
     llm_rpm: Optional[PositiveInt] = None
     """The requests per minute of the LLM model."""
 
+    llm_presence_penalty: Optional[PositiveFloat] = None
+    """The presence penalty of the LLM model."""
+
+    llm_frequency_penalty: Optional[PositiveFloat] = None
+    """The frequency penalty of the LLM model."""
+
     embedding_api_endpoint: Optional[HttpUrl] = None
     """The OpenAI API endpoint."""
 

fabricatio/models/kwargs_types.py

@@ -1,47 +1,16 @@
 """This module contains the types for the keyword arguments of the methods in the models module."""
 
-from importlib.util import find_spec
-from typing import Any, Dict, List, Optional, Required, TypedDict
+from typing import Any, Dict, List, NotRequired, Optional, Required, TypedDict
 
 from litellm.caching.caching import CacheMode
 from litellm.types.caching import CachingSupportedCallTypes
 
-if find_spec("pymilvus"):
-    from pymilvus import CollectionSchema
-    from pymilvus.milvus_client import IndexParams
 
-    class CollectionConfigKwargs(TypedDict, total=False):
-        """Configuration parameters for a vector collection.
+class ChunkKwargs(TypedDict):
+    """Configuration parameters for chunking operations."""
 
-        These arguments are typically used when configuring connections to vector databases.
-        """
-
-        dimension: int | None
-        primary_field_name: str
-        id_type: str
-        vector_field_name: str
-        metric_type: str
-        timeout: float | None
-        schema: CollectionSchema | None
-        index_params: IndexParams | None
-
-
-class FetchKwargs(TypedDict, total=False):
-    """Arguments for fetching data from vector collections.
-
-    Controls how data is retrieved from vector databases, including filtering
-    and result limiting parameters.
-    """
-
-    collection_name: str | None
-    similarity_threshold: float
-    result_per_query: int
-
-
-class RetrievalKwargs(FetchKwargs, total=False):
-    """Arguments for retrieval operations."""
-
-    final_limit: int
+    max_chunk_size: int
+    max_overlapping_rate: NotRequired[float]
 
 
 class EmbeddingKwargs(TypedDict, total=False):
@@ -76,6 +45,8 @@ class LLMKwargs(TypedDict, total=False):
     no_store: bool  # If store the response of this call to cache
     cache_ttl: int  # how long the stored cache is alive, in seconds
     s_maxage: int  # max accepted age of cached response, in seconds
+    presence_penalty: float
+    frequency_penalty: float
 
 
 class GenerateKwargs(LLMKwargs, total=False):
@@ -139,6 +110,7 @@ class ReviewKwargs[T](ReviewInnerKwargs[T], total=False):
 
 class ReferencedKwargs[T](ValidateKwargs[T], total=False):
     """Arguments for content review operations."""
+
     reference: str
 
 

fabricatio/models/task.py

@@ -7,11 +7,11 @@ from asyncio import Queue
 from typing import Any, List, Optional, Self
 
 from fabricatio.config import configs
+from fabricatio.constants import TaskStatus
 from fabricatio.core import env
 from fabricatio.journal import logger
 from fabricatio.models.events import Event, EventLike
 from fabricatio.models.generic import ProposedAble, WithBriefing, WithDependency
-from fabricatio.models.utils import TaskStatus
 from fabricatio.rust_instances import TEMPLATE_MANAGER
 from pydantic import Field, PrivateAttr
 
@@ -112,12 +112,12 @@ class Task[T](WithBriefing, ProposedAble, WithDependency):
         """Return a formatted status label for the task.
 
         Args:
-            status (TaskStatus): The status of the task.
+            status (fabricatio.constants.TaskStatus): The status of the task.
 
         Returns:
             str: The formatted status label.
         """
-        return self._namespace.derive(self.name).push(status.value).collapse()
+        return self._namespace.derive(self.name).push(status).collapse()
 
     @property
     def pending_label(self) -> str:

fabricatio/models/usages.py

@@ -2,7 +2,7 @@
 
 import traceback
 from asyncio import gather
-from typing import Callable, Dict, Iterable, List, Optional, Self, Sequence, Set, Union, Unpack, overload
+from typing import Callable, Dict, Iterable, List, Literal, Optional, Self, Sequence, Set, Union, Unpack, overload
 
 import asyncstdlib
 import litellm
@@ -13,7 +13,6 @@ from fabricatio.models.generic import ScopedConfig, WithBriefing
 from fabricatio.models.kwargs_types import ChooseKwargs, EmbeddingKwargs, GenerateKwargs, LLMKwargs, ValidateKwargs
 from fabricatio.models.task import Task
 from fabricatio.models.tool import Tool, ToolBox
-from fabricatio.models.utils import Messages
 from fabricatio.parser import GenericCapture, JsonCapture
 from fabricatio.rust_instances import TEMPLATE_MANAGER
 from fabricatio.utils import ok
@@ -28,7 +27,7 @@ from litellm.types.utils import (
 )
 from litellm.utils import CustomStreamWrapper, token_counter  # pyright: ignore [reportPrivateImportUsage]
 from more_itertools import duplicates_everseen
-from pydantic import Field, NonNegativeInt, PositiveInt
+from pydantic import BaseModel, ConfigDict, Field, NonNegativeInt, PositiveInt
 
 if configs.cache.enabled and configs.cache.type:
     litellm.enable_cache(type=configs.cache.type, **configs.cache.params)
@@ -64,7 +63,7 @@ class LLMUsage(ScopedConfig):
         self._added_deployment = ROUTER.upsert_deployment(deployment)
         return ROUTER
 
-    # noinspection PyTypeChecker,PydanticTypeChecker
+    # noinspection PyTypeChecker,PydanticTypeChecker,t
     async def aquery(
         self,
         messages: List[Dict[str, str]],
@@ -123,6 +122,12 @@ class LLMUsage(ScopedConfig):
                 "cache-ttl": kwargs.get("cache_ttl"),
                 "s-maxage": kwargs.get("s_maxage"),
             },
+            presence_penalty=kwargs.get("presence_penalty")
+            or self.llm_presence_penalty
+            or configs.llm.presence_penalty,
+            frequency_penalty=kwargs.get("frequency_penalty")
+            or self.llm_frequency_penalty
+            or configs.llm.frequency_penalty,
         )
 
     async def ainvoke(
@@ -299,10 +304,11 @@ class LLMUsage(ScopedConfig):
             for lap in range(max_validations):
                 try:
                     if ((validated := validator(response := await self.aask(question=q, **kwargs))) is not None) or (
-                        co_extractor
+                        co_extractor is not None
+                        and logger.debug("Co-extraction is enabled.") is None
                         and (
                             validated := validator(
-                                await self.aask(
+                                response := await self.aask(
                                     question=(
                                         TEMPLATE_MANAGER.render_template(
                                             configs.templates.co_validation_template,
@@ -319,12 +325,13 @@ class LLMUsage(ScopedConfig):
                         return validated
 
                 except RateLimitError as e:
-                    logger.warning(f"Rate limit error: {e}")
+                    logger.warning(f"Rate limit error:\n{e}")
                     continue
                 except Exception as e:  # noqa: BLE001
-                    logger.error(f"Error during validation: \n{e}")
+                    logger.error(f"Error during validation:\n{e}")
                     logger.debug(traceback.format_exc())
                     break
+                logger.error(f"Failed to validate the response at {lap}th attempt:\n{response}")
                 if not kwargs.get("no_cache"):
                     kwargs["no_cache"] = True
                     logger.debug("Closed the cache for the next attempt")
@@ -493,7 +500,7 @@ class LLMUsage(ScopedConfig):
         affirm_case: str = "",
         deny_case: str = "",
         **kwargs: Unpack[ValidateKwargs[bool]],
-    ) -> bool:
+    ) -> Optional[bool]:
         """Asynchronously judges a prompt using AI validation.
 
         Args:
@@ -730,3 +737,72 @@ class ToolBoxUsage(LLMUsage):
         for other in (x for x in others if isinstance(x, ToolBoxUsage)):
             other.toolboxes.update(self.toolboxes)
         return self
+
+
+class Message(BaseModel):
+    """A class representing a message."""
+
+    model_config = ConfigDict(use_attribute_docstrings=True)
+    role: Literal["user", "system", "assistant"]
+    """The role of the message sender."""
+    content: str
+    """The content of the message."""
+
+
+class Messages(list):
+    """A list of messages."""
+
+    def add_message(self, role: Literal["user", "system", "assistant"], content: str) -> Self:
+        """Adds a message to the list with the specified role and content.
+
+        Args:
+            role (Literal["user", "system", "assistant"]): The role of the message sender.
+            content (str): The content of the message.
+
+        Returns:
+            Self: The current instance of Messages to allow method chaining.
+        """
+        if content:
+            self.append(Message(role=role, content=content))
+        return self
+
+    def add_user_message(self, content: str) -> Self:
+        """Adds a user message to the list with the specified content.
+
+        Args:
+            content (str): The content of the user message.
+
+        Returns:
+            Self: The current instance of Messages to allow method chaining.
+        """
+        return self.add_message("user", content)
+
+    def add_system_message(self, content: str) -> Self:
+        """Adds a system message to the list with the specified content.
+
+        Args:
+            content (str): The content of the system message.
+
+        Returns:
+            Self: The current instance of Messages to allow method chaining.
+        """
+        return self.add_message("system", content)
+
+    def add_assistant_message(self, content: str) -> Self:
+        """Adds an assistant message to the list with the specified content.
+
+        Args:
+            content (str): The content of the assistant message.
+
+        Returns:
+            Self: The current instance of Messages to allow method chaining.
+        """
+        return self.add_message("assistant", content)
+
+    def as_list(self) -> List[Dict[str, str]]:
+        """Converts the messages to a list of dictionaries.
+
+        Returns:
+            list[dict]: A list of dictionaries representing the messages.
+        """
+        return [message.model_dump() for message in self]

fabricatio/parser.py

@@ -48,10 +48,10 @@ class Capture(BaseModel):
             case "json" if configs.general.use_json_repair:
                 logger.debug("Applying json repair to text.")
                 if isinstance(text, str):
-                    return repair_json(text, ensure_ascii=False)
-                return [repair_json(item, ensure_ascii=False) for item in text]
+                    return repair_json(text, ensure_ascii=False)  # pyright: ignore [reportReturnType]
+                return [repair_json(item, ensure_ascii=False) for item in text]  # pyright: ignore [reportReturnType, reportGeneralTypeIssues]
             case _:
-                return text
+                return text  # pyright: ignore [reportReturnType]
 
     def capture(self, text: str) -> Tuple[str, ...] | str | None:
         """Capture the first occurrence of the pattern in the given text.
@@ -88,7 +88,7 @@ class Capture(BaseModel):
         if (cap := self.capture(text)) is None:
             return None
         try:
-            return convertor(cap)
+            return convertor(cap)  # pyright: ignore [reportArgumentType]
         except (ValueError, SyntaxError, ValidationError) as e:
             logger.error(f"Failed to convert text using {convertor.__name__} to convert.\nerror: {e}\n {cap}")
             return None
@@ -120,7 +120,7 @@ class Capture(BaseModel):
             judges.append(lambda output_obj: len(output_obj) == length)
 
         if (out := self.convert_with(text, deserializer)) and all(j(out) for j in judges):
-            return out
+            return out  # pyright: ignore [reportReturnType]
         return None
 
     @classmethod

fabricatio/rust.pyi

@@ -1,5 +1,4 @@
-"""
-Python interface definitions for Rust-based functionality.
+"""Python interface definitions for Rust-based functionality.
 
 This module provides type stubs and documentation for Rust-implemented utilities,
 including template rendering, cryptographic hashing, language detection, and
@@ -12,11 +11,8 @@ Key Features:
 - Text utilities: Word boundary splitting and word counting.
 """
 
-
 from pathlib import Path
-from typing import List, Optional
-
-from pydantic import JsonValue
+from typing import Any, Dict, List, Optional
 
 
 class TemplateManager:
@@ -29,7 +25,7 @@ class TemplateManager:
     """
 
     def __init__(
-        self, template_dirs: List[Path], suffix: Optional[str] = None, active_loading: Optional[bool] = None
+            self, template_dirs: List[Path], suffix: Optional[str] = None, active_loading: Optional[bool] = None
     ) -> None:
         """Initialize the template manager.
 
@@ -59,7 +55,7 @@ class TemplateManager:
         This refreshes the template cache, finding any new or modified templates.
         """
 
-    def render_template(self, name: str, data: JsonValue) -> str:
+    def render_template(self, name: str, data: Dict[str, Any]) -> str:
         """Render a template with context data.
 
         Args:
@@ -73,7 +69,7 @@ class TemplateManager:
             RuntimeError: If template rendering fails
         """
 
-    def render_template_raw(self, template: str, data: JsonValue) -> str:
+    def render_template_raw(self, template: str, data: Dict[str, Any]) -> str:
         """Render a template with context data.
 
         Args:
@@ -84,6 +80,7 @@ class TemplateManager:
             Rendered template content as string
         """
 
+
 def blake3_hash(content: bytes) -> str:
     """Calculate the BLAKE3 cryptographic hash of data.
 
@@ -94,6 +91,7 @@ def blake3_hash(content: bytes) -> str:
         Hex-encoded BLAKE3 hash string
     """
 
+
 def detect_language(string: str) -> str:
     """Detect the language of a given string."""
 
@@ -107,6 +105,32 @@ def split_word_bounds(string: str) -> List[str]:
     Returns:
         A list of words extracted from the string.
     """
+
+
+def split_sentence_bounds(string: str) -> List[str]:
+    """Split the string into sentences based on sentence boundaries.
+
+    Args:
+        string: The input string to be split.
+
+    Returns:
+        A list of sentences extracted from the string.
+    """
+
+
+def split_into_chunks(string: str, max_chunk_size: int, max_overlapping_rate: float = 0.3) -> List[str]:
+    """Split the string into chunks of a specified size.
+
+    Args:
+        string: The input string to be split.
+        max_chunk_size: The maximum size of each chunk.
+        max_overlapping_rate: The minimum overlapping rate between chunks.
+
+    Returns:
+        A list of chunks extracted from the string.
+    """
+
+
 def word_count(string: str) -> int:
     """Count the number of words in the string.
 
@@ -118,6 +142,98 @@ def word_count(string: str) -> int:
     """
 
 
+def is_chinese(string: str) -> bool:
+    """Check if the given string is in Chinese."""
+
+
+def is_english(string: str) -> bool:
+    """Check if the given string is in English."""
+
+
+def is_japanese(string: str) -> bool:
+    """Check if the given string is in Japanese."""
+
+
+def is_korean(string: str) -> bool:
+    """Check if the given string is in Korean."""
+
+
+def is_arabic(string: str) -> bool:
+    """Check if the given string is in Arabic."""
+
+
+def is_russian(string: str) -> bool:
+    """Check if the given string is in Russian."""
+
+
+def is_german(string: str) -> bool:
+    """Check if the given string is in German."""
+
+
+def is_french(string: str) -> bool:
+    """Check if the given string is in French."""
+
+
+def is_hindi(string: str) -> bool:
+    """Check if the given string is in Hindi."""
+
+
+def is_italian(string: str) -> bool:
+    """Check if the given string is in Italian."""
+
+
+def is_dutch(string: str) -> bool:
+    """Check if the given string is in Dutch."""
+
+
+def is_portuguese(string: str) -> bool:
+    """Check if the given string is in Portuguese."""
+
+
+def is_swedish(string: str) -> bool:
+    """Check if the given string is in Swedish."""
+
+
+def is_turkish(string: str) -> bool:
+    """Check if the given string is in Turkish."""
+
+
+def is_vietnamese(string: str) -> bool:
+    """Check if the given string is in Vietnamese."""
+
+
+def tex_to_typst(string: str) -> str:
+    """Convert TeX to Typst.
+
+    Args:
+        string: The input TeX string to be converted.
+
+    Returns:
+        The converted Typst string.
+    """
+
+
+def convert_all_inline_tex(string: str) -> str:
+    """Convert all inline TeX code in the string.
+
+    Args:
+        string: The input string containing inline TeX code wrapped in $code$.
+
+    Returns:
+        The converted string with inline TeX code replaced.
+    """
+
+
+def convert_all_block_tex(string: str) -> str:
+    """Convert all block TeX code in the string.
+
+    Args:
+        string: The input string containing block TeX code wrapped in $$code$$.
+
+    Returns:
+        The converted string with block TeX code replaced.
+    """
+
 
 class BibManager:
     """BibTeX bibliography manager for parsing and querying citation data."""
@@ -132,7 +248,7 @@ class BibManager:
             RuntimeError: If file cannot be read or parsed
         """
 
-    def get_cite_key(self, title: str) -> Optional[str]:
+    def get_cite_key_by_title(self, title: str) -> Optional[str]:
         """Find citation key by exact title match.
 
         Args:
@@ -142,6 +258,16 @@ class BibManager:
             Citation key if exact match found, None otherwise
         """
 
+    def get_cite_key_by_title_fuzzy(self, title: str) -> Optional[str]:
+        """Find citation key by fuzzy title match.
+
+        Args:
+            title: Search term to find in bibliography entries
+
+        Returns:
+            Citation key of best matching entry, or None if no good match
+        """
+
     def get_cite_key_fuzzy(self, query: str) -> Optional[str]:
         """Find best matching citation using fuzzy text search.
 
@@ -195,6 +321,7 @@ class BibManager:
         Returns:
             Abstract if found, None otherwise
         """
+
     def get_title_by_key(self, key: str) -> Optional[str]:
         """Retrieve the title by citation key.