fabricatio 0.2.4.dev1__cp312-cp312-manylinux_2_34_x86_64.whl → 0.2.4.dev3__cp312-cp312-manylinux_2_34_x86_64.whl

fabricatio/__init__.py

@@ -3,7 +3,7 @@
 from importlib.util import find_spec
 
 from fabricatio._rust_instances import template_manager
-from fabricatio.actions import ExtractArticleEssence
+from fabricatio.actions.article import ExtractArticleEssence
 from fabricatio.core import env
 from fabricatio.fs import magika
 from fabricatio.journal import logger
@@ -44,6 +44,8 @@ __all__ = [
 
 
 if find_spec("pymilvus"):
+    from fabricatio.actions.rag import InjectToDB
     from fabricatio.capabilities.rag import RAG
+    from fabricatio.workflows.articles import StoreArticle
 
-    __all__ += ["RAG"]
+    __all__ += ["RAG", "InjectToDB", "StoreArticle"]

fabricatio/actions/article.py

@@ -11,11 +11,16 @@ from fabricatio.models.task import Task
 
 
 class ExtractArticleEssence(Action):
-    """Extract the essence of article(s)."""
+    """Extract the essence of article(s) in text format from the paths specified in the task dependencies.
+
+    Notes:
+        This action is designed to extract vital information from articles with Markdown format, which is pure text, and
+        which is converted from pdf files using `magic-pdf` from the `MinerU` project, see https://github.com/opendatalab/MinerU
+    """
 
     name: str = "extract article essence"
     """The name of the action."""
-    description: str = "Extract the essence of an article. output as json"
+    description: str = "Extract the essence of article(s) from the paths specified in the task dependencies."
     """The description of the action."""
 
     output_key: str = "article_essence"

fabricatio/actions/rag.py

@@ -0,0 +1,25 @@
+"""Inject data into the database."""
+
+from typing import List, Optional, Unpack
+
+from fabricatio.capabilities.rag import RAG
+from fabricatio.models.action import Action
+from fabricatio.models.generic import PrepareVectorization
+
+
+class InjectToDB(Action, RAG):
+    """Inject data into the database."""
+
+    output_key: str = "collection_name"
+
+    async def _execute[T: PrepareVectorization](
+        self, to_inject: T | List[T], collection_name: Optional[str] = "my_collection", **cxt: Unpack
+    ) -> str:
+        if not isinstance(to_inject, list):
+            to_inject = [to_inject]
+
+        await self.view(collection_name, create=True).consume_string(
+            [t.prepare_vectorization(self.embedding_max_sequence_length) for t in to_inject],
+        )
+
+        return collection_name

fabricatio/capabilities/rag.py

@@ -13,10 +13,16 @@ from typing import Any, Callable, Dict, List, Optional, Self, Union, Unpack, ove
 from fabricatio._rust_instances import template_manager
 from fabricatio.config import configs
 from fabricatio.journal import logger
-from fabricatio.models.kwargs_types import CollectionSimpleConfigKwargs, EmbeddingKwargs, FetchKwargs, LLMKwargs
+from fabricatio.models.kwargs_types import (
+    ChooseKwargs,
+    CollectionSimpleConfigKwargs,
+    EmbeddingKwargs,
+    FetchKwargs,
+    LLMKwargs,
+)
 from fabricatio.models.usages import EmbeddingUsage
 from fabricatio.models.utils import MilvusData
-from more_itertools.recipes import flatten
+from more_itertools.recipes import flatten, unique
 from pydantic import Field, PrivateAttr
 
 
@@ -221,9 +227,9 @@ class RAG(EmbeddingUsage):
 
         # Step 2: Flatten the search results
         flattened_results = flatten(search_results)
-
+        unique_results = unique(flattened_results, key=itemgetter("id"))
         # Step 3: Sort by distance (descending)
-        sorted_results = sorted(flattened_results, key=itemgetter("distance"), reverse=True)
+        sorted_results = sorted(unique_results, key=itemgetter("distance"), reverse=True)
 
         logger.debug(f"Searched similarities: {[t['distance'] for t in sorted_results]}")
         # Step 4: Extract the entities
@@ -308,3 +314,21 @@ class RAG(EmbeddingUsage):
             f"{rendered}\n\n{extra_system_message}",
             **kwargs,
         )
+
+    async def arefined_query(self, question: List[str] | str, **kwargs: Unpack[ChooseKwargs]) -> List[str]:
+        """Refines the given question using a template.
+
+        Args:
+            question (List[str] | str): The question to be refined.
+            **kwargs (Unpack[ChooseKwargs]): Additional keyword arguments for the refinement process.
+
+        Returns:
+            List[str]: A list of refined questions.
+        """
+        return await self.aliststr(
+            template_manager.render_template(
+                configs.templates.refined_query_template,
+                {"question": [question] if isinstance(question, str) else question},
+            ),
+            **kwargs,
+        )

fabricatio/config.py

@@ -209,6 +209,9 @@ class TemplateConfig(BaseModel):
     liststr_template: str = Field(default="liststr")
     """The name of the liststr template which will be used to display a list of strings."""
 
+    refined_query_template: str = Field(default="refined_query")
+    """The name of the refined query template which will be used to refine a query."""
+
 
 class MagikaConfig(BaseModel):
     """Magika configuration class."""

fabricatio/fs/__init__.py

@@ -1,5 +1,17 @@
 """FileSystem manipulation module for Fabricatio."""
 
-from fabricatio.fs.readers import magika
+from fabricatio.fs.curd import copy_file, create_directory, delete_directory, delete_file, dump_text, move_file, tree
+from fabricatio.fs.readers import magika, safe_json_read, safe_text_read
 
-__all__ = ["magika"]
+__all__ = [
+    "copy_file",
+    "create_directory",
+    "delete_directory",
+    "delete_file",
+    "dump_text",
+    "magika",
+    "move_file",
+    "safe_json_read",
+    "safe_text_read",
+    "tree",
+]

fabricatio/models/action.py

@@ -3,7 +3,7 @@
 import traceback
 from abc import abstractmethod
 from asyncio import Queue, create_task
-from typing import Any, Dict, Self, Tuple, Type, Union, Unpack
+from typing import Any, Dict, Self, Tuple, Type, Union, Unpack, final
 
 from fabricatio.capabilities.rating import GiveRating
 from fabricatio.capabilities.task import HandleTask, ProposeTask
@@ -17,11 +17,26 @@ from pydantic import Field, PrivateAttr
 class Action(HandleTask, ProposeTask, GiveRating):
     """Class that represents an action to be executed in a workflow."""
 
+    name: str = Field(default="")
+    """The name of the action."""
+    description: str = Field(default="")
+    """The description of the action."""
     personality: str = Field(default="")
     """The personality of whom the action belongs to."""
     output_key: str = Field(default="")
     """The key of the output data."""
 
+    @final
+    def model_post_init(self, __context: Any) -> None:
+        """Initialize the action by setting the name if not provided.
+
+        Args:
+            __context: The context to be used for initialization.
+        """
+        self.name = self.name or self.__class__.__name__
+
+        self.description = self.description or self.__class__.__doc__ or ""
+
     @abstractmethod
     async def _execute(self, **cxt: Unpack) -> Any:
         """Execute the action with the provided arguments.
@@ -34,6 +49,7 @@ class Action(HandleTask, ProposeTask, GiveRating):
         """
         pass
 
+    @final
     async def act(self, cxt: Dict[str, Any]) -> Dict[str, Any]:
         """Perform the action by executing it and setting the output data.
 

fabricatio/workflows/articles.py

@@ -0,0 +1,11 @@
+"""Store article essence in the database."""
+
+from fabricatio.actions.article import ExtractArticleEssence
+from fabricatio.actions.rag import InjectToDB
+from fabricatio.models.action import WorkFlow
+
+StoreArticle = WorkFlow(
+    name="Extract Article Essence",
+    description="Extract the essence of an article in the given path, and store it in the database.",
+    steps=(ExtractArticleEssence(output_key="to_inject"), InjectToDB(output_key="task_output")),
+)

{fabricatio-0.2.4.dev1.dist-info → fabricatio-0.2.4.dev3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fabricatio
-Version: 0.2.4.dev1
+Version: 0.2.4.dev3
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: 3.12

{fabricatio-0.2.4.dev1.dist-info → fabricatio-0.2.4.dev3.dist-info}/RECORD

@@ -1,6 +1,6 @@
-fabricatio-0.2.4.dev1.dist-info/METADATA,sha256=Y5iwc8ttCDQZinkAsi9fgqDCvHLpE91wIha-thu9NJ0,8589
-fabricatio-0.2.4.dev1.dist-info/WHEEL,sha256=RIvmwLDYujv60MYBx2jxyP4vdn1DD7X0kBgz1TQvZuc,108
-fabricatio-0.2.4.dev1.dist-info/licenses/LICENSE,sha256=yDZaTLnOi03bi3Dk6f5IjhLUc5old2yOsihHWU0z-i0,1067
+fabricatio-0.2.4.dev3.dist-info/METADATA,sha256=aL1eAIQviM4LPVRJ4D2HyZT87tbZq47zgG9n8Q7xCzY,8589
+fabricatio-0.2.4.dev3.dist-info/WHEEL,sha256=RIvmwLDYujv60MYBx2jxyP4vdn1DD7X0kBgz1TQvZuc,108
+fabricatio-0.2.4.dev3.dist-info/licenses/LICENSE,sha256=yDZaTLnOi03bi3Dk6f5IjhLUc5old2yOsihHWU0z-i0,1067
 fabricatio/decorators.py,sha256=cJHsxxbnMhc4SzPl4454CPLuDP3H0qbTrzV_U2rLPrs,6372
 fabricatio/core.py,sha256=MaEKZ6DDmbdScAY-7F1gwGA6fr7ADX6Mz5rNVi2msFA,6277
 fabricatio/models/generic.py,sha256=bjAD8g4tz1VjnIfnpTD6sRu1dBuBoswmUrn6pCnL6aY,11504
@@ -12,7 +12,7 @@ fabricatio/models/utils.py,sha256=vahILaesw50ofFft-wZ9kZ_Qogqi6vIOISWczvwVXxk,43
 fabricatio/models/usages.py,sha256=iXh4MbDhrzshOxH704NY_5nbt7Lx5ckyBCs_nboxlM8,24943
 fabricatio/models/events.py,sha256=sBCKeNoYc4TFDoke-jhFEyA11RcdGu-wdF5ynAuVOMM,3983
 fabricatio/models/task.py,sha256=A0xdbrMeqhWqpmoMplTuaZmsKKHl93InZJ2iKjaRNBA,10683
-fabricatio/models/action.py,sha256=EfggvxZFhdfHmie0C3XCHFXSlLIjZKB3MXf0530MAiI,5803
+fabricatio/models/action.py,sha256=CYg4YPOo_NhAl8Nm8bCoEJkF4xPZZ-ZsVJmABrx2u_Q,6335
 fabricatio/toolboxes/fs.py,sha256=oZjGOmtN8ZbkMXxCMatqvdPavVXyQ87AEDc1lx9jimY,483
 fabricatio/toolboxes/__init__.py,sha256=iEBsH0aRPLccIe6eyZ6iH5GdK8yYNQTnCwLzw4pWXAA,465
 fabricatio/toolboxes/arithmetic.py,sha256=sSTPkKI6-mb278DwQKFO9jKyzc9kCx45xNH7V6bGBpE,1307
@@ -20,19 +20,20 @@ fabricatio/toolboxes/task.py,sha256=9J3W-48CCEUh5PFz0XEC47Hv23Ugn1BEg-J_woQE1UA,
 fabricatio/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fabricatio/fs/readers.py,sha256=lqMrM5Nt3l6QJmPDoBno2PSaN2BbFwfUjBqaghrbK3A,1002
 fabricatio/fs/curd.py,sha256=g74Pc2WPSkPlCXX0cWj2jyiDYTfkLwpwM480eyCkmc8,3851
-fabricatio/fs/__init__.py,sha256=bYE9r8uR0dtknzbg_YaGv_6Wwa27ntkQt0Tl7Kb3HFI,117
-fabricatio/config.py,sha256=ZUUJ5rb6ZBq8Vrkk9wqfMU8ncomRoHgarW7YuThEJK4,13188
+fabricatio/fs/__init__.py,sha256=Jlzk4AvBFgcgQ6nIgGRSUIh2UZHtpl27eyu395lR3c0,448
+fabricatio/config.py,sha256=DfGJurqWVoLNgx9-DiCqBhcxgMSBY52atriw2iZsq1E,13341
 fabricatio/journal.py,sha256=bzxZay48ZWI0VIkkDXm4Wc_Cc9lBQYa2VGx3Hxy_PtA,753
-fabricatio/__init__.py,sha256=I1CtksFFUCXsnSLAL51369S4ExY9uUE9Xdo4jK6hFls,1356
-fabricatio/actions/__init__.py,sha256=ggTW1wZntPjWP-JHEZe1vqoyImmUnCt15h40zC5rx98,125
-fabricatio/actions/article.py,sha256=esuDgD1eJ4W_UPvpdHSPugT2qDbqfNdatFx9xgFT5l4,1610
+fabricatio/__init__.py,sha256=gakP6hzHIQccKu9ByHhUP-cN36wCPEr11s13iTdO6EY,1503
+fabricatio/actions/rag.py,sha256=tP1uKLq7mo19YhsJzFbGYryrcMx6xcvBc3n_Wzh0hjg,796
+fabricatio/actions/article.py,sha256=ozqEa75_MS4OfCJZzcmF5_s3AJmAUrQGdHORWtZBsE0,1977
 fabricatio/_rust_instances.py,sha256=JAtO-vL8ihvduf1SHLNf0w7ZSVGCJeIv6zZ9Ekyy1hY,271
+fabricatio/workflows/articles.py,sha256=KlTaMSnxkQqHAIl-zZ8_wMqheka7IL99nt7XzQxDSF0,462
 fabricatio/parser.py,sha256=S-4p1mjvJZEgnQW6WKdkmE68MzqqWXGSiLbADdhE-W4,4791
-fabricatio/capabilities/rag.py,sha256=bF-SSZRQbG00QjzoAXrWCfzFxHxfvxV3al_G8T4jxL8,12985
+fabricatio/capabilities/rag.py,sha256=OscJ8MVgOViMYps7rp8-MzAfSiog5i7zp7RkFzoSJPI,13805
 fabricatio/capabilities/rating.py,sha256=E1obAS48B4j3Cf6rC6h-TyzRb4vQ2Hho2IM7630jSoI,13695
 fabricatio/capabilities/propose.py,sha256=oW9QKpY2mDkVPEvgsgqxXxDR2ylVqN5TEL8E0Wh_vJI,1714
 fabricatio/capabilities/task.py,sha256=BISAFbMgBI4udW0otE3-iyq0RXc05YN_30N0yI5qLxQ,4504
 fabricatio/_rust.pyi,sha256=clhcURuiB9zlFo4m3VyoWQ8Xs4tvg6KNHXpF-ok9h4o,1703
-fabricatio/_rust.cpython-312-x86_64-linux-gnu.so,sha256=CDyz4oafWsO46rOc8DyHBfDVj4dY2LkheAQTGfSXmgs,1333952
-fabricatio-0.2.4.dev1.data/scripts/tdown,sha256=lUUKFHytXf1um4ZhtzvZlMqpBPWN7IjAsKKT9IgDwog,4560064
-fabricatio-0.2.4.dev1.dist-info/RECORD,,
+fabricatio/_rust.cpython-312-x86_64-linux-gnu.so,sha256=cdj3XswppApEvlov50l7ZgS_MLO3DMj_8B2SFdsWXe0,1350776
+fabricatio-0.2.4.dev3.data/scripts/tdown,sha256=lUUKFHytXf1um4ZhtzvZlMqpBPWN7IjAsKKT9IgDwog,4560064
+fabricatio-0.2.4.dev3.dist-info/RECORD,,

fabricatio/actions/__init__.py

@@ -1,5 +0,0 @@
-"""module for actions."""
-
-from fabricatio.actions.article import ExtractArticleEssence
-
-__all__ = ["ExtractArticleEssence"]