fabricatio 0.3.15.dev5__cp312-cp312-win_amd64.whl → 0.4.5.dev0__cp312-cp312-win_amd64.whl

fabricatio/capabilities/correct.py

@@ -1,228 +0,0 @@
-"""A module containing the Correct capability for reviewing, validating, and improving objects."""
-
-from abc import ABC
-from asyncio import gather
-from typing import Optional, Type, Unpack, cast
-
-from fabricatio.capabilities.propose import Propose
-from fabricatio.capabilities.rating import Rating
-from fabricatio.journal import logger
-from fabricatio.models.adv_kwargs_types import CorrectKwargs
-from fabricatio.models.extra.problem import Improvement, ProblemSolutions
-from fabricatio.models.generic import ProposedUpdateAble, SketchedAble
-from fabricatio.models.kwargs_types import (
-    BestKwargs,
-    ValidateKwargs,
-)
-from fabricatio.rust import CONFIG, TEMPLATE_MANAGER
-from fabricatio.utils import fallback_kwargs, ok, override_kwargs
-
-
-class Correct(Rating, Propose, ABC):
-    """A class that provides the capability to correct objects."""
-
-    async def decide_solution(
-        self, problem_solutions: ProblemSolutions, **kwargs: Unpack[BestKwargs]
-    ) -> ProblemSolutions:
-        """Decide the best solution from a list of problem solutions.
-
-        Args:
-            problem_solutions (ProblemSolutions): The problem solutions to evaluate.
-            **kwargs (Unpack[BestKwargs]): Additional keyword arguments for the decision process.
-
-        Returns:
-            ProblemSolutions: The problem solutions with the best solution selected.
-        """
-        if (leng := len(problem_solutions.solutions)) == 0:
-            logger.error(f"No solutions found in ProblemSolutions, Skip: `{problem_solutions.problem.name}`")
-        if leng > 1:
-            logger.info(f"{leng} solutions found in Problem `{problem_solutions.problem.name}`, select the best.")
-            problem_solutions.solutions = await self.best(problem_solutions.solutions, **kwargs)
-        return problem_solutions
-
-    async def decide_improvement(self, improvement: Improvement, **kwargs: Unpack[BestKwargs]) -> Improvement:
-        """Decide the best solution for each problem solution in an improvement.
-
-        Args:
-            improvement (Improvement): The improvement containing problem solutions to evaluate.
-            **kwargs (Unpack[BestKwargs]): Additional keyword arguments for the decision process.
-
-        Returns:
-            Improvement: The improvement with the best solutions selected for each problem solution.
-        """
-        if leng := len(improvement.problem_solutions):
-            logger.debug(f"{leng} problem_solutions found in Improvement, decide solution for each of them.")
-            await gather(
-                *[
-                    self.decide_solution(
-                        ps,
-                        **fallback_kwargs(
-                            kwargs, topic=f"which solution is better to deal this problem {ps.problem.description}\n\n"
-                        ),
-                    )
-                    for ps in improvement.problem_solutions
-                ],
-            )
-            if any(not (violated := ps).decided() for ps in improvement.problem_solutions):
-                logger.error(f"Some problem_solutions are not decided: {violated}")
-            else:
-                logger.success(f"All problem_solutions are decided '{improvement.focused_on}'")
-        else:
-            logger.error(f"No problem_solutions found in Improvement, Skip: {improvement}")
-        return improvement
-
-    async def fix_troubled_obj[M: SketchedAble](
-        self,
-        obj: M,
-        problem_solutions: ProblemSolutions,
-        reference: str = "",
-        **kwargs: Unpack[ValidateKwargs[M]],
-    ) -> Optional[M]:
-        """Fix a troubled object based on problem solutions.
-
-        Args:
-            obj (M): The object to be fixed.
-            problem_solutions (ProblemSolutions): The problem solutions to apply.
-            reference (str): A reference or contextual information for the object.
-            **kwargs (Unpack[ValidateKwargs[M]]): Additional keyword arguments for the validation process.
-
-        Returns:
-            Optional[M]: The fixed object, or None if fixing fails.
-        """
-        return await self.propose(
-            cast("Type[M]", obj.__class__),
-            TEMPLATE_MANAGER.render_template(
-                CONFIG.templates.fix_troubled_obj_template,
-                {
-                    "problem": problem_solutions.problem.display(),
-                    "solution": ok(
-                        problem_solutions.final_solution(),
-                        f"{len(problem_solutions.solutions)} solution Found for `{problem_solutions.problem.name}`.",
-                    ).display(),
-                    "reference": reference,
-                },
-            ),
-            **kwargs,
-        )
-
-    async def fix_troubled_string(
-        self,
-        input_text: str,
-        problem_solutions: ProblemSolutions,
-        reference: str = "",
-        **kwargs: Unpack[ValidateKwargs[str]],
-    ) -> Optional[str]:
-        """Fix a troubled string based on problem solutions.
-
-        Args:
-            input_text (str): The string to be fixed.
-            problem_solutions (ProblemSolutions): The problem solutions to apply.
-            reference (str): A reference or contextual information for the string.
-            **kwargs (Unpack[ValidateKwargs[str]]): Additional keyword arguments for the validation process.
-
-        Returns:
-            Optional[str]: The fixed string, or None if fixing fails.
-        """
-        return await self.ageneric_string(
-            TEMPLATE_MANAGER.render_template(
-                CONFIG.templates.fix_troubled_string_template,
-                {
-                    "problem": problem_solutions.problem.display(),
-                    "solution": ok(
-                        problem_solutions.final_solution(),
-                        f"No solution found for problem: {problem_solutions.problem}",
-                    ).display(),
-                    "reference": reference,
-                    "string_to_fix": input_text,
-                },
-            ),
-            **kwargs,
-        )
-
-    async def correct_obj[M: SketchedAble](
-        self,
-        obj: M,
-        improvement: Improvement,
-        reference: str = "",
-        **kwargs: Unpack[ValidateKwargs[M]],
-    ) -> Optional[M]:
-        """Review and correct an object based on defined criteria and templates.
-
-        This method first conducts a review of the given object, then uses the review results
-        to generate a corrected version of the object using appropriate templates.
-
-        Args:
-            obj (M): The object to be reviewed and corrected. Must implement ProposedAble.
-            improvement (Improvement): The improvement object containing the review results.
-            reference (str): A reference or contextual information for the object.
-            **kwargs (Unpack[ValidateKwargs[M]]): Review configuration parameters including criteria and review options.
-
-        Returns:
-            Optional[M]: A corrected version of the input object, or None if correction fails.
-
-        Raises:
-            TypeError: If the provided object doesn't implement Display or WithBriefing interfaces.
-        """
-        if not improvement.decided():
-            logger.info(f"Improvement {improvement.focused_on} not decided, start deciding...")
-            improvement = await self.decide_improvement(improvement, **override_kwargs(kwargs, default=None))
-
-        total = len(improvement.problem_solutions)
-        for idx, ps in enumerate(improvement.problem_solutions):
-            logger.info(f"[{idx + 1}/{total}] Fixing {obj.__class__.__name__} for problem `{ps.problem.name}`")
-            fixed_obj = await self.fix_troubled_obj(obj, ps, reference, **kwargs)
-            if fixed_obj is None:
-                logger.error(f"[{idx + 1}/{total}] Failed to fix problem `{ps.problem.name}`")
-                return None
-            obj = fixed_obj
-        return obj
-
-    async def correct_string(
-        self, input_text: str, improvement: Improvement, reference: str = "", **kwargs: Unpack[ValidateKwargs[str]]
-    ) -> Optional[str]:
-        """Review and correct a string based on defined criteria and templates.
-
-        This method first conducts a review of the given string, then uses the review results
-        to generate a corrected version of the string using appropriate templates.
-
-        Args:
-            input_text (str): The string to be reviewed and corrected.
-            improvement (Improvement): The improvement object containing the review results.
-            reference (str): A reference or contextual information for the string.
-            **kwargs (Unpack[ValidateKwargs[str]]): Review configuration parameters including criteria and review options.
-
-        Returns:
-            Optional[str]: A corrected version of the input string, or None if correction fails.
-        """
-        if not improvement.decided():
-            logger.info(f"Improvement {improvement.focused_on} not decided, start deciding...")
-
-            improvement = await self.decide_improvement(improvement, **override_kwargs(kwargs, default=None))
-
-        for ps in improvement.problem_solutions:
-            fixed_string = await self.fix_troubled_string(input_text, ps, reference, **kwargs)
-            if fixed_string is None:
-                logger.error(
-                    f"Failed to fix troubling string when deal with problem: {ps.problem}",
-                )
-                return None
-            input_text = fixed_string
-        return input_text
-
-    async def correct_obj_inplace[M: ProposedUpdateAble](
-        self, obj: M, **kwargs: Unpack[CorrectKwargs[M]]
-    ) -> Optional[M]:
-        """Correct an object in place based on defined criteria and templates.
-
-        Args:
-            obj (M): The object to be corrected.
-            **kwargs (Unpack[CorrectKwargs[M]]): Additional keyword arguments for the correction process.
-
-        Returns:
-            Optional[M]: The corrected object, or None if correction fails.
-        """
-        corrected_obj = await self.correct_obj(obj, **kwargs)
-        if corrected_obj is None:
-            return corrected_obj
-        obj.update_from(corrected_obj)
-        return obj

fabricatio/capabilities/extract.py

@@ -1,74 +0,0 @@
-"""A module that provide capabilities for extracting information from a given source to a model."""
-
-from abc import ABC
-from typing import List, Optional, Type, Unpack, overload
-
-from fabricatio import TEMPLATE_MANAGER
-from fabricatio.capabilities.propose import Propose
-from fabricatio.models.generic import ProposedAble
-from fabricatio.models.kwargs_types import ValidateKwargs
-from fabricatio.rust import CONFIG
-
-
-class Extract(Propose, ABC):
-    """A class that extract information from a given source to a model."""
-
-    @overload
-    async def extract[M: ProposedAble](
-        self,
-        cls: Type[M],
-        source: str,
-        extract_requirement: Optional[str] = None,
-        align_language: bool = True,
-        **kwargs: Unpack[ValidateKwargs[M]],
-    ) -> M: ...
-
-    @overload
-    async def extract[M: ProposedAble](
-        self,
-        cls: Type[M],
-        source: str,
-        extract_requirement: Optional[str] = None,
-        align_language: bool = True,
-        **kwargs: Unpack[ValidateKwargs[None]],
-    ) -> Optional[M]: ...
-
-    @overload
-    async def extract[M: ProposedAble](
-        self,
-        cls: Type[M],
-        source: List[str],
-        extract_requirement: Optional[str] = None,
-        align_language: bool = True,
-        **kwargs: Unpack[ValidateKwargs[M]],
-    ) -> List[M]: ...
-
-    @overload
-    async def extract[M: ProposedAble](
-        self,
-        cls: Type[M],
-        source: List[str],
-        extract_requirement: Optional[str] = None,
-        align_language: bool = True,
-        **kwargs: Unpack[ValidateKwargs[None]],
-    ) -> List[Optional[M]]: ...
-
-    async def extract[M: ProposedAble](
-        self,
-        cls: Type[M],
-        source: List[str] | str,
-        extract_requirement: Optional[str] = None,
-        align_language: bool = True,
-        **kwargs: Unpack[ValidateKwargs[Optional[M]]],
-    ) -> M | List[M] | Optional[M] | List[Optional[M]]:
-        """Extract information from a given source to a model."""
-        return await self.propose(
-            cls,
-            prompt=TEMPLATE_MANAGER.render_template(
-                CONFIG.templates.extract_template,
-                [{"source": s, "extract_requirement": extract_requirement} for s in source]
-                if isinstance(source, list)
-                else {"source": source, "extract_requirement": extract_requirement, "align_language": align_language},
-            ),
-            **kwargs,
-        )

fabricatio/capabilities/propose.py

@@ -1,65 +0,0 @@
-"""A module for the task capabilities of the Fabricatio library."""
-from abc import ABC
-from typing import List, Optional, Type, Unpack, overload
-
-from fabricatio.models.generic import ProposedAble
-from fabricatio.models.kwargs_types import ValidateKwargs
-from fabricatio.models.usages import LLMUsage
-
-
-class Propose(LLMUsage,ABC):
-    """A class that proposes an Obj based on a prompt."""
-
-    @overload
-    async def propose[M: ProposedAble](
-        self,
-        cls: Type[M],
-        prompt: List[str],
-        **kwargs: Unpack[ValidateKwargs[None]],
-    ) -> List[Optional[M]]: ...
-
-    @overload
-    async def propose[M: ProposedAble](
-        self,
-        cls: Type[M],
-        prompt: List[str],
-        **kwargs: Unpack[ValidateKwargs[M]],
-    ) -> List[M]: ...
-
-    @overload
-    async def propose[M: ProposedAble](
-        self,
-        cls: Type[M],
-        prompt: str,
-        **kwargs: Unpack[ValidateKwargs[None]],
-    ) -> Optional[M]: ...
-    @overload
-    async def propose[M: ProposedAble](
-        self,
-        cls: Type[M],
-        prompt: str,
-        **kwargs: Unpack[ValidateKwargs[M]],
-    ) -> M: ...
-
-    async def propose[M: ProposedAble](
-        self,
-        cls: Type[M],
-        prompt: List[str] | str,
-        **kwargs: Unpack[ValidateKwargs[Optional[M]]],
-    ) -> Optional[M] | List[Optional[M]] | M | List[M]:
-        """Asynchronously proposes a task based on a given prompt and parameters.
-
-        Parameters:
-            cls: The class type of the task to be proposed.
-            prompt: The prompt text for proposing a task, which is a string that must be provided.
-            **kwargs: The keyword arguments for the LLM (Large Language Model) usage.
-
-        Returns:
-            A Task object based on the proposal result.
-        """
-        return await self.aask_validate(
-            question=cls.create_json_prompt(prompt),
-            validator=cls.instantiate_from_string,
-            **kwargs,
-        )
-

fabricatio/capabilities/rag.py

@@ -1,264 +0,0 @@
-"""A module for the RAG (Retrieval Augmented Generation) model."""
-
-from abc import ABC
-
-try:
-    from pymilvus import MilvusClient
-except ImportError as e:
-    raise RuntimeError(
-        "pymilvus is not installed. Have you installed `fabricatio[rag]` instead of `fabricatio`?"
-    ) from e
-from functools import lru_cache
-from operator import itemgetter
-from typing import List, Optional, Self, Type, Unpack
-
-from more_itertools.recipes import flatten, unique
-from pydantic import Field, PrivateAttr
-
-from fabricatio.journal import logger
-from fabricatio.models.adv_kwargs_types import CollectionConfigKwargs, FetchKwargs
-from fabricatio.models.extra.rag import MilvusDataBase
-from fabricatio.models.kwargs_types import ChooseKwargs
-from fabricatio.models.usages import EmbeddingUsage
-from fabricatio.rust import CONFIG, TEMPLATE_MANAGER
-from fabricatio.utils import ok
-
-
-@lru_cache(maxsize=None)
-def create_client(uri: str, token: str = "", timeout: Optional[float] = None) -> MilvusClient:
-    """Create a Milvus client."""
-    return MilvusClient(
-        uri=uri,
-        token=token,
-        timeout=timeout,
-    )
-
-
-class RAG(EmbeddingUsage, ABC):
-    """A class representing the RAG (Retrieval Augmented Generation) model."""
-
-    target_collection: Optional[str] = Field(default=None)
-    """The name of the collection being viewed."""
-
-    _client: Optional[MilvusClient] = PrivateAttr(None)
-    """The Milvus client used for the RAG model."""
-
-    @property
-    def client(self) -> MilvusClient:
-        """Return the Milvus client."""
-        if self._client is None:
-            raise RuntimeError("Client is not initialized. Have you called `self.init_client()`?")
-        return self._client
-
-    def init_client(
-            self,
-            milvus_uri: Optional[str] = None,
-            milvus_token: Optional[str] = None,
-            milvus_timeout: Optional[float] = None,
-    ) -> Self:
-        """Initialize the Milvus client."""
-        self._client = create_client(
-            uri=milvus_uri or ok(self.milvus_uri or CONFIG.rag.milvus_uri),
-            token=milvus_token
-                  or (token.get_secret_value() if (token := (self.milvus_token or CONFIG.rag.milvus_token)) else ""),
-            timeout=milvus_timeout or self.milvus_timeout or CONFIG.rag.milvus_timeout,
-        )
-        return self
-
-    def check_client(self, init: bool = True) -> Self:
-        """Check if the client is initialized, and if not, initialize it."""
-        if self._client is None and init:
-            return self.init_client()
-        if self._client is None and not init:
-            raise RuntimeError("Client is not initialized. Have you called `self.init_client()`?")
-        return self
-
-    def view(
-            self, collection_name: Optional[str], create: bool = False, **kwargs: Unpack[CollectionConfigKwargs]
-    ) -> Self:
-        """View the specified collection.
-
-        Args:
-            collection_name (str): The name of the collection.
-            create (bool): Whether to create the collection if it does not exist.
-            **kwargs (Unpack[CollectionConfigKwargs]): Additional keyword arguments for collection configuration.
-        """
-        if create and collection_name and not self.check_client().client.has_collection(collection_name):
-            kwargs["dimension"] = ok(
-                kwargs.get("dimension")
-                or self.milvus_dimensions
-                or CONFIG.rag.milvus_dimensions
-                or self.embedding_dimensions
-                or CONFIG.embedding.dimensions,
-                "`dimension` is not set at any level.",
-            )
-            self.client.create_collection(collection_name, auto_id=True, **kwargs)
-            logger.info(f"Creating collection {collection_name}")
-
-        self.target_collection = collection_name
-        return self
-
-    def quit_viewing(self) -> Self:
-        """Quit the current view.
-
-        Returns:
-            Self: The current instance, allowing for method chaining.
-        """
-        return self.view(None)
-
-    @property
-    def safe_target_collection(self) -> str:
-        """Get the name of the collection being viewed, raise an error if not viewing any collection.
-
-        Returns:
-            str: The name of the collection being viewed.
-        """
-        return ok(self.target_collection, "No collection is being viewed. Have you called `self.view()`?")
-
-    async def add_document[D: MilvusDataBase](
-            self, data: List[D] | D, collection_name: Optional[str] = None, flush: bool = False
-    ) -> Self:
-        """Adds a document to the specified collection.
-
-        Args:
-            data (Union[Dict[str, Any], MilvusDataBase] | List[Union[Dict[str, Any], MilvusDataBase]]): The data to be added to the collection.
-            collection_name (Optional[str]): The name of the collection. If not provided, the currently viewed collection is used.
-            flush (bool): Whether to flush the collection after insertion.
-
-        Returns:
-            Self: The current instance, allowing for method chaining.
-        """
-        if isinstance(data, MilvusDataBase):
-            data = [data]
-
-        data_vec = await self.vectorize([d.prepare_vectorization() for d in data])
-        prepared_data = [d.prepare_insertion(vec) for d, vec in zip(data, data_vec, strict=True)]
-
-        c_name = collection_name or self.safe_target_collection
-        self.check_client().client.insert(c_name, prepared_data)
-
-        if flush:
-            logger.debug(f"Flushing collection {c_name}")
-            self.client.flush(c_name)
-        return self
-
-    async def afetch_document[D: MilvusDataBase](
-            self,
-            query: List[str],
-            document_model: Type[D],
-            collection_name: Optional[str] = None,
-            similarity_threshold: float = 0.37,
-            result_per_query: int = 10,
-            tei_endpoint: Optional[str] = None,
-            reranker_threshold: float = 0.7,
-            filter_expr: str = "",
-    ) -> List[D]:
-        """Asynchronously fetches documents from a Milvus database based on input vectors.
-
-        Args:
-           query (List[str]): A list of vectors to search for in the database.
-           document_model (Type[D]): The model class used to convert fetched data into document objects.
-           collection_name (Optional[str]): The name of the collection to search within.
-                                             If None, the currently viewed collection is used.
-           similarity_threshold (float): The similarity threshold for vector search. Defaults to 0.37.
-           result_per_query (int): The maximum number of results to return per query. Defaults to 10.
-           tei_endpoint (str): the endpoint of the TEI api.
-           reranker_threshold (float): The threshold used to filtered low relativity document.
-           filter_expr (str) : The filter expression used to filter out unwanted documents.
-
-        Returns:
-           List[D]: A list of document objects created from the fetched data.
-        """
-        # Step 1: Search for vectors
-        search_results = self.check_client().client.search(
-            collection_name or self.safe_target_collection,
-            await self.vectorize(query),
-            search_params={"radius": similarity_threshold},
-            output_fields=list(document_model.model_fields),
-            filter=filter_expr,
-            limit=result_per_query,
-        )
-        if tei_endpoint is not None:
-            from fabricatio.rust import TEIClient
-
-            reranker = TEIClient(base_url=tei_endpoint)
-
-            retrieved_id = set()
-            raw_result = []
-
-            for q, g in zip(query, search_results, strict=True):
-                models = document_model.from_sequence([res["entity"] for res in g if res["id"] not in retrieved_id])
-                logger.debug(f"Retrived {len(g)} raw document, filtered out {len(models)}.")
-                retrieved_id.update(res["id"] for res in g)
-                if not models:
-                    continue
-                rank_scores = await reranker.arerank(q, [m.prepare_vectorization() for m in models], truncate=True,
-                                                     truncation_direction="Left")
-                raw_result.extend((models[idx], scr) for (idx, scr) in rank_scores if scr > reranker_threshold)
-
-            raw_result_sorted = sorted(raw_result, key=lambda x: x[1], reverse=True)
-            return [r[0] for r in raw_result_sorted]
-
-        # Step 2: Flatten the search results
-        flattened_results = flatten(search_results)
-        unique_results = unique(flattened_results, key=itemgetter("id"))
-
-        # Step 3: Sort by distance (descending)
-        sorted_results = sorted(unique_results, key=itemgetter("distance"), reverse=True)
-
-        logger.debug(
-            f"Fetched {len(sorted_results)} document,searched similarities: {[t['distance'] for t in sorted_results]}"
-        )
-        # Step 4: Extract the entities
-        resp = [result["entity"] for result in sorted_results]
-
-        return document_model.from_sequence(resp)
-
-    async def aretrieve[D: MilvusDataBase](
-            self,
-            query: List[str] | str,
-            document_model: Type[D],
-            max_accepted: int = 20,
-            **kwargs: Unpack[FetchKwargs],
-    ) -> List[D]:
-        """Retrieve data from the collection.
-
-        Args:
-            query (List[str] | str): The query to be used for retrieval.
-            document_model (Type[D]): The model class used to convert retrieved data into document objects.
-            max_accepted (int): The final limit on the number of results to return.
-            **kwargs (Unpack[FetchKwargs]): Additional keyword arguments for retrieval.
-
-        Returns:
-            List[D]: A list of document objects created from the retrieved data.
-        """
-        if isinstance(query, str):
-            query = [query]
-
-        return (
-                   await self.afetch_document(
-                       query=query,
-                       document_model=document_model,
-                       **kwargs,
-                   )
-               )[:max_accepted]
-
-    async def arefined_query(
-            self, question: List[str] | str, **kwargs: Unpack[ChooseKwargs[Optional[List[str]]]]
-    ) -> Optional[List[str]]:
-        """Refines the given question using a template.
-
-        Args:
-            question (List[str] | str): The question to be refined.
-            **kwargs (Unpack[ChooseKwargs]): Additional keyword arguments for the refinement process.
-
-        Returns:
-            List[str]: A list of refined questions.
-        """
-        return await self.alist_str(
-            TEMPLATE_MANAGER.render_template(
-                CONFIG.templates.refined_query_template,
-                {"question": [question] if isinstance(question, str) else question},
-            ),
-            **kwargs,
-        )