hamtaa-texttools 0.1.43__py3-none-any.whl

texttools/tools/question_generator/gemma_question_generator.py

@@ -0,0 +1,198 @@
+from typing import Any, Optional
+
+from openai import OpenAI
+
+from texttools.base.base_question_generator import BaseQuestionGenerator
+from texttools.formatter import Gemma3Formatter
+
+# class QuestionGeneration(BaseModel):
+#     generated_question: str
+
+
+class GemmaQuestionGenerator(BaseQuestionGenerator):
+    """
+    Question Generator for Gemma-style models with optional reasoning step.
+    Outputs JSON with a single string field: {"generated_question": "..."}.
+
+    Allows optional extra instructions via `prompt_template`.
+    """
+
+    def __init__(
+        self,
+        client: OpenAI,
+        *,
+        model: str,
+        chat_formatter: Optional[Any] = None,
+        use_reason: bool = False,
+        temperature: float = 0.0,
+        prompt_template: Optional[str] = None,
+        handlers: Optional[list[Any]] = None,
+        **client_kwargs: Any,
+    ):
+        super().__init__(handlers)
+        self.client = client
+        self.model = model
+        self.temperature = temperature
+        self.client_kwargs = client_kwargs
+
+        self.chat_formatter = chat_formatter or Gemma3Formatter()
+
+        self.use_reason = use_reason
+        self.prompt_template = prompt_template
+
+        # Define the JSON schema for the generated question output
+        self.json_schema = {"generated_question": "string"}
+
+    def _build_messages(
+        self, answer: str, reason: Optional[str] = None
+    ) -> list[dict[str, str]]:
+        """
+        Builds the message list for the LLM API call for question generation.
+        """
+        clean_answer = self.preprocess(answer)
+        messages: list[dict[str, str]] = []
+
+        if self.prompt_template:
+            messages.append({"role": "user", "content": self.prompt_template})
+
+        if reason:
+            messages.append(
+                {
+                    "role": "user",
+                    "content": f"Based on this analysis of the answer: {reason}",
+                }
+            )
+
+        messages.append(
+            {
+                "role": "user",
+                "content": """Given the following answer, generate a single, 
+                appropriate question that this answer would directly respond to.
+                the generated answer should be independently meaningful,
+                and not mentioning any verbs like, this, that, he or she ... on the question.
+                # **the generated question will be in the language of the users input**
+                
+                """,
+            }
+        )
+        messages.append(
+            {"role": "user", "content": f"here is the text: {clean_answer}"}
+        )
+
+        # Ensure the schema is dumped as a valid JSON string for the LLM
+        # schema_instr = f"Respond only in JSON format: {json.dumps(self.json_schema)}"
+        messages.append(
+            {
+                "role": "user",
+                "content": """
+        Respond only with the new generated question, without any additional information.
+        **the generated question will be in the language of the users input**
+                         """,
+            }
+        )
+
+        # messages.append(
+        #     {"role": "assistant", "content": "{\n"}
+        # )  # Hint to start JSON output
+        # in this new version we will use
+        # parse function of openai library
+
+        # this line will restructure the messages
+        # based on the formatter that we provided
+        # some models will require custom settings
+        restructured = self.chat_formatter.format(messages=messages)
+
+        return restructured
+
+    def _reason(self, answer: str) -> str:
+        """
+        Internal reasoning step to help the model understand the core information
+        and implications of the answer.
+        """
+        messages = [
+            {
+                "role": "user",
+                "content": """
+                    Analyze the following answer to identify its key facts,
+                    main subject, and what kind of information it provides.
+                    Provide a brief, summarized understanding of the answer's content that will 
+                    help in formulating a relevant and direct question.
+                    
+                    provide the summary in the language of the content.
+                    just mention the keypoints that was provided in the answer
+                    
+                    
+                    """,
+            },
+            {
+                "role": "user",
+                "content": f"""
+                
+                    Here is the content:
+                    
+                    {answer}
+                    
+                    respond only with the language of the content
+                    """,
+            },
+        ]
+
+        restructured = self.chat_formatter.format(messages=messages)
+
+        resp = self.client.chat.completions.create(
+            model=self.model,
+            messages=restructured,
+            temperature=self.temperature,
+            **self.client_kwargs,
+        )
+
+        reason_summary = resp.choices[0].message.content.strip()
+        return reason_summary
+
+    def generate_question(self, answer: str) -> str:
+        """
+        Generates a question for the input `answer`.
+        Optionally uses an internal reasoning step for better accuracy.
+        """
+        reason_summary = None
+        if self.use_reason:
+            reason_summary = self._reason(answer)
+
+        messages = self._build_messages(answer, reason_summary)
+
+        # i am deprecating the usage of structured output in the tasks that
+        # the input and output is str
+        # as we have noticed a huge decrease in the models outputs quality
+
+        #
+        # completion = self.client.beta.chat.completions.parse(
+        #     model=self.model,
+        #     messages=messages,
+        #     response_format=QuestionGeneration,
+        #     temperature=self.temperature,
+        #     extra_body=dict(guided_decoding_backend="outlines"),
+        #     **self.client_kwargs,
+        # )
+        # message = completion.choices[0].message
+        # if message.parsed:
+        #     result = message.parsed.generated_question
+        # else:
+        #     raise ValueError(f"Failed to parse the response. Raw content: {message.content}")
+
+        resp = self.client.chat.completions.create(
+            model=self.model,
+            messages=messages,
+            temperature=self.temperature,
+            **self.client_kwargs,
+        )
+
+        result = resp.choices[0].message.content.strip()
+
+        # dispatch and return
+        self._dispatch(
+            {
+                "original_answer": answer,
+                "generated_question": result,
+            }
+        )
+        return result

texttools/tools/reranker/__init__.py

@@ -0,0 +1,3 @@
+from .reranker import GemmaReranker
+from .scorer import GemmaScorer
+from .sorter import GemmaSorter

texttools/tools/reranker/reranker.py

@@ -0,0 +1,137 @@
+import logging
+from typing import Any, Optional
+
+from openai import OpenAI
+
+from texttools.base.base_task_performer import BaseTaskPerformer
+
+from .scorer import GemmaScorer
+from .sorter import GemmaSorter
+
+
+class GemmaReranker(BaseTaskPerformer):
+    """
+    A Reranker component that orchestrates a GemmaScorer and GemmaSorter
+    to refine the order of a list of search results based on a query.
+    It first scores individual results and then sorts them.
+    """
+
+    def __init__(
+        self,
+        client: OpenAI,
+        *,
+        model: str,
+        scorer_prompt_template: Optional[str] = None,
+        sorter_prompt_template: Optional[str] = None,
+        scorer_use_reason: bool = False,
+        sorter_use_reason: bool = False,
+        temperature: float = 0.0,
+        handlers: Optional[list[Any]] = None,
+        **client_kwargs: Any,
+    ):
+        """
+        Initializes the GemmaReranker with configurations for its internal scorer and sorter.
+
+        :param client: An initialized OpenAI client (or compatible).
+        :param model: The LLM model to use for both scoring and sorting (e.g., "gemma-7b-it").
+        :param scorer_prompt_template: Optional initial system-level prompt for the scorer.
+        :param sorter_prompt_template: Optional initial system-level prompt for the sorter.
+        :param scorer_use_reason: If True, the internal scorer will use an internal reasoning step.
+        :param sorter_use_reason: If True, the internal sorter will use an internal reasoning step.
+        :param temperature: The sampling temperature for LLM generation (0.0 for deterministic).
+        :param handlers: Optional list of handlers for dispatching reranking results.
+        :param client_kwargs: Additional keyword arguments for the OpenAI client.
+        """
+        super().__init__(handlers)
+        self.client = client
+        self.model = model
+        self.temperature = temperature
+        self.client_kwargs = client_kwargs
+
+        # Initialize the internal Scorer and Sorter components
+        self.scorer = GemmaScorer(
+            client=self.client,
+            model=self.model,
+            temperature=self.temperature,
+            prompt_template=scorer_prompt_template,
+            use_reason=scorer_use_reason,
+            handlers=[],
+            **self.client_kwargs,
+        )
+        self.sorter = GemmaSorter(
+            client=self.client,
+            model=self.model,
+            temperature=self.temperature,
+            prompt_template=sorter_prompt_template,
+            use_reason=sorter_use_reason,
+            handlers=[],
+            **self.client_kwargs,
+        )
+
+    def perform(
+        self, query: str, results: list[dict[str, Any]]
+    ) -> list[dict[str, Any]]:
+        """
+        Performs the complete reranking operation: scoring each result and then sorting them.
+
+        :param query: The original search query.
+        :param results: A list of result dictionaries to be reranked.
+                        Each dictionary should ideally have a 'text' key.
+        :return: A list of result dictionaries, reranked by relevance.
+                 Each dictionary will have an added 'score' key.
+        :raises ValueError: If the input results list is empty or if sub-components fail.
+        """
+        if not results:
+            logging.info(
+                "Received empty list of results for reranking. Returning empty list."
+            )
+            self._dispatch(
+                {"query": query, "original_results": results, "reranked_results": []}
+            )
+            return []
+
+        scored_results: list[dict[str, Any]] = []
+
+        # Step 1: Score each individual result
+        for i, res in enumerate(results):
+            # Create a unique internal ID if not already present
+            _internal_id = res.get("id", f"rerank_id_{i}")
+
+            result_text = res.get("text")
+            if not result_text:
+                logging.warning(
+                    f"Result with ID '{_internal_id}' has no 'text' key. Skipping scoring."
+                )
+                score = 0  # Default score if no text
+            else:
+                try:
+                    score = self.scorer.perform(query, result_text)
+                except Exception as e:
+                    logging.error(
+                        f"Scorer failed for result ID '{_internal_id}': {e}",
+                        exc_info=True,
+                    )
+                    score = 0  # Default score on scorer failure
+
+            # Create a new dictionary to add the score and internal ID
+            # It's important to copy the original result and add these for the sorter.
+            scored_res_copy = res.copy()
+            scored_res_copy["score"] = score
+            scored_res_copy["_internal_id"] = _internal_id  # Sorter expects this
+            scored_results.append(scored_res_copy)
+
+        # Step 2: Sort the scored results
+        # The sorter's perform method expects a list of dictionaries with '_internal_id' and 'score'
+        reranked_results = self.sorter.perform(query, scored_results)
+
+        # Dispatch the final reranked results
+        self._dispatch(
+            {
+                "query": query,
+                "original_results": results,
+                "scored_results": scored_results,  # Can include internal IDs for debugging if needed
+                "reranked_results": reranked_results,  # This will have '_internal_id' removed by sorter
+            }
+        )
+
+        return reranked_results

texttools/tools/reranker/scorer.py

@@ -0,0 +1,216 @@
+import json
+import logging
+from typing import Any, Optional
+
+from openai import OpenAI
+
+from texttools.base.base_task_performer import BaseTaskPerformer
+
+
+class GemmaScorer(BaseTaskPerformer):
+    """
+    A scorer component utilizing Gemma-style LLMs to evaluate the relevance of
+    individual text results against a given query. It assigns a score from 0-5.
+    Can optionally include a reasoning step for each result to enhance accuracy.
+    """
+
+    def __init__(
+        self,
+        client: OpenAI,
+        *,
+        model: str,
+        temperature: float = 0.0,
+        prompt_template: Optional[str] = None,
+        use_reason: bool = False,
+        handlers: Optional[list[Any]] = None,
+        **client_kwargs: Any,
+    ):
+        """
+        Initializes the GemmaScorer.
+
+        :param client: An initialized OpenAI client (or compatible).
+        :param model: The name of the LLM model to use for scoring (e.g., "gemma-7b-it").
+        :param temperature: The sampling temperature for LLM generation (0.0 for deterministic).
+        :param prompt_template: An optional initial system-level prompt for the LLM.
+        :param use_reason: If True, the scorer will perform an internal reasoning step
+                           for each result and include it in the scoring prompt.
+        :param handlers: Optional list of handlers for dispatching scoring results.
+        :param client_kwargs: Additional keyword arguments for the OpenAI client.
+        """
+        super().__init__(handlers)
+        self.client = client
+        self.model = model
+        self.temperature = temperature
+        self.client_kwargs = client_kwargs
+        self.prompt_template = prompt_template
+        self.use_reason = use_reason
+
+        # Defines the expected JSON schema for the LLM's score output.
+        self.score_schema = {"score": "integer"}
+
+    def _build_messages(
+        self, query: str, result_text: str, reason: Optional[str] = None
+    ) -> list[dict[str, str]]:
+        """
+        Constructs the messages payload for the LLM API call to score a single result.
+        Now includes an optional 'reason' parameter.
+
+        :param query: The search query.
+        :param result_text: The text content of the result to be scored.
+        :param reason: An optional reasoning summary generated internally for this specific result.
+        :return: A list of message dictionaries formatted for the LLM API.
+        """
+        clean_query = self._preprocess(query)
+        clean_result_text = self._preprocess(result_text)
+
+        messages: list[dict[str, str]] = []
+
+        if self.prompt_template:
+            messages.append({"role": "user", "content": self.prompt_template})
+
+        # Conditionally add the reason to the prompt
+        if self.use_reason and reason:
+            messages.append(
+                {
+                    "role": "user",
+                    "content": f"Consider this preliminary analysis for scoring: {reason}",
+                }
+            )
+
+        messages.append({"role": "user", "content": f"Query: {clean_query}"})
+        messages.append(
+            {"role": "user", "content": f"Result text to score: {clean_result_text}"}
+        )
+
+        scoring_instruction = (
+            "Score this result from 0 (not relevant) to 5 (highly relevant) based on how well it matches the query. "
+            "Return only the score as a JSON object with a 'score' key."
+        )
+        messages.append({"role": "user", "content": scoring_instruction})
+
+        schema_instr = f"Respond only in JSON format: {json.dumps(self.score_schema)}"
+        messages.append({"role": "user", "content": schema_instr})
+
+        messages.append({"role": "assistant", "content": "{"})
+        return messages
+
+    def _reason(self, query: str, result_text: str) -> str:
+        """
+        Generates a brief reasoning summary for a single result's relevance to the query.
+        This summary is intended to provide additional context to the LLM for scoring.
+
+        :param query: The search query.
+        :param result_text: The text content of the result being analyzed.
+        :return: A string containing the reasoning summary.
+        """
+        clean_query = self._preprocess(query)
+        clean_result_text = self._preprocess(result_text)
+
+        # Improved handling for result text truncation
+        display_result_text = clean_result_text
+        if len(clean_result_text) > 200:
+            display_result_text = clean_result_text[:200] + "..."
+
+        reason_prompt = f"""
+            Analyze the relevance of the following result text to the given query.
+            Focus on key terms, concepts, and overall intent.
+            Query: "{clean_query}"
+            Result Text: "{display_result_text}"
+            
+            Provide a brief summary of why this result might or might not be relevant to the query.
+            """
+
+        messages = [
+            {"role": "user", "content": reason_prompt},
+        ]
+
+        resp = self.client.chat.completions.create(
+            model=self.model,
+            messages=messages,
+            temperature=self.temperature,
+            **self.client_kwargs,
+        )
+        return resp.choices[0].message.content.strip()
+
+    def perform(self, query: str, result_text: str) -> int:
+        """
+        Scores a single result's text against a given query using the configured LLM.
+        This is the main public method for the scorer.
+
+        :param query: The search query string.
+        :param result_text: The text content of the result to be scored.
+        :return: An integer score (0-5) representing relevance.
+        :raises ValueError: If result_text is empty, LLM output is malformed, or score is invalid.
+        """
+        if not result_text:
+            logging.warning("Received empty result text for scoring.")
+            self._dispatch(
+                {
+                    "query": query,
+                    "result_text": result_text,
+                    "score": 0,
+                    "error": "Empty result text",
+                }
+            )
+            return 0
+
+        # Generate reason if enabled
+        reason_for_scoring = None
+        if self.use_reason:
+            reason_for_scoring = self._reason(query, result_text)
+
+        messages = self._build_messages(query, result_text, reason_for_scoring)
+
+        resp = self.client.chat.completions.create(
+            model=self.model,
+            messages=messages,
+            temperature=self.temperature,
+            **self.client_kwargs,
+        )
+        raw_score_output = resp.choices[0].message.content.strip()
+
+        # Robustly extract JSON if LLM adds preamble
+        if not raw_score_output.startswith("{"):
+            raw_score_output = "{" + raw_score_output
+        try:
+            parsed_score = json.loads(raw_score_output)
+        except json.JSONDecodeError as e:
+            logging.error(
+                f"Failed to parse JSON for single result scoring: {e}\nRaw output: {raw_score_output}"
+            )
+            self._dispatch(
+                {
+                    "query": query,
+                    "result_text": result_text,
+                    "score": 0,
+                    "error": f"JSON parsing failed: {e}",
+                }
+            )
+            return 0
+
+        score = parsed_score.get("score")
+        if not isinstance(score, (int, float)) or not (0 <= score <= 5):
+            logging.warning(
+                f"LLM returned invalid score for result text '{result_text[:50]}...': {score}. "
+                "Expected integer 0-5. Raw: {raw_score_output}"
+            )
+            self._dispatch(
+                {
+                    "query": query,
+                    "result_text": result_text,
+                    "score": 0,
+                    "warning": "Invalid score format",
+                }
+            )
+            return 0
+
+        final_score = int(score)
+        self._dispatch(
+            {
+                "query": query,
+                "result_text": result_text,
+                "score": final_score,
+                "reason": reason_for_scoring,
+            }
+        )
+        return final_score