chatterer 0.1.6__py3-none-any.whl → 0.1.8__py3-none-any.whl

chatterer/__init__.py

@@ -1,4 +1,10 @@
 from .language_model import Chatterer
+from .messages import (
+    AIMessage,
+    BaseMessage,
+    HumanMessage,
+    SystemMessage,
+)
 from .strategies import (
     AoTPipeline,
     AoTPrompter,
@@ -7,6 +13,7 @@ from .strategies import (
 )
 from .tools import (
     anything_to_markdown,
+    citation_chunker,
     get_default_html_to_markdown_options,
     html_to_markdown,
     pdf_to_text,
@@ -24,4 +31,9 @@ __all__ = [
     "pdf_to_text",
     "get_default_html_to_markdown_options",
     "pyscripts_to_snippets",
+    "citation_chunker",
+    "BaseMessage",
+    "HumanMessage",
+    "SystemMessage",
+    "AIMessage",
 ]

chatterer/language_model.py

@@ -8,15 +8,18 @@ from typing import (
     Type,
     TypeAlias,
     TypeVar,
+    cast,
+    overload,
 )
 
 from langchain_core.language_models.base import LanguageModelInput
 from langchain_core.language_models.chat_models import BaseChatModel
-from langchain_core.messages import HumanMessage
 from langchain_core.runnables.base import Runnable
 from langchain_core.runnables.config import RunnableConfig
 from pydantic import BaseModel, Field
 
+from .messages import AIMessage, BaseMessage, HumanMessage
+
 if TYPE_CHECKING:
     from instructor import Partial
 
@@ -32,19 +35,37 @@ class Chatterer(BaseModel):
     client: BaseChatModel
     structured_output_kwargs: dict[str, Any] = Field(default_factory=dict)
 
-    def __call__(self, messages: LanguageModelInput) -> str:
-        """
-        Generate text from the given input messages.
-
-        Args:
-            messages (LanguageModelInput): Input messages for the language model.
-                Can be one of the following types:
-                - str: A single string message.
-                - list[dict[str, str]]: A list of dictionaries with 'role' and 'content' keys.
-                - tuple[str, str]: A tuple of strings representing the role and content of a single message.
-                - list[BaseMessage]: A list of BaseMessage objects. (BaseMessage is a Pydantic model; e.g. can import AIMessage, HumanMessage, SystemMessage from langchain_core.messages)
-        """
-        return self.generate(messages)
+    @overload
+    def __call__(
+        self,
+        messages: LanguageModelInput,
+        response_model: Type[PydanticModelT],
+        config: Optional[RunnableConfig] = None,
+        stop: Optional[list[str]] = None,
+        **kwargs: Any,
+    ) -> PydanticModelT: ...
+
+    @overload
+    def __call__(
+        self,
+        messages: LanguageModelInput,
+        response_model: None = None,
+        config: Optional[RunnableConfig] = None,
+        stop: Optional[list[str]] = None,
+        **kwargs: Any,
+    ) -> str: ...
+
+    def __call__(
+        self,
+        messages: LanguageModelInput,
+        response_model: Optional[Type[PydanticModelT]] = None,
+        config: Optional[RunnableConfig] = None,
+        stop: Optional[list[str]] = None,
+        **kwargs: Any,
+    ) -> str | PydanticModelT:
+        if response_model:
+            return self.generate_pydantic(response_model, messages, config, stop, **kwargs)
+        return self.client.invoke(input=messages, config=config, stop=stop, **kwargs).text()
 
     @classmethod
     def openai(
@@ -233,6 +254,40 @@ class Chatterer(BaseModel):
             )
         ])
 
+    @staticmethod
+    def get_num_tokens_from_message(message: BaseMessage) -> Optional[tuple[int, int]]:
+        try:
+            if isinstance(message, AIMessage) and (usage_metadata := message.usage_metadata):
+                input_tokens = int(usage_metadata["input_tokens"])
+                output_tokens = int(usage_metadata["output_tokens"])
+            else:
+                # Dynamic extraction for unknown structures
+                input_tokens: Optional[int] = None
+                output_tokens: Optional[int] = None
+
+                def _find_tokens(obj: object) -> None:
+                    nonlocal input_tokens, output_tokens
+                    if isinstance(obj, dict):
+                        for key, value in cast(dict[object, object], obj).items():
+                            if isinstance(value, int):
+                                if "input" in str(key) or "prompt" in str(key):
+                                    input_tokens = value
+                                elif "output" in str(key) or "completion" in str(key):
+                                    output_tokens = value
+                            else:
+                                _find_tokens(value)
+                    elif isinstance(obj, list):
+                        for item in cast(list[object], obj):
+                            _find_tokens(item)
+
+                _find_tokens(message.model_dump())
+
+            if input_tokens is None or output_tokens is None:
+                return None
+            return input_tokens, output_tokens
+        except Exception:
+            return None
+
 
 def with_structured_output(
     client: BaseChatModel,
@@ -255,36 +310,25 @@ if __name__ == "__main__":
 
     # === Synchronous Tests ===
 
-    # 1. generate
+    # generate
     print("=== Synchronous generate ===")
-    result_sync = chatterer.generate(prompt)
+    result_sync = chatterer(prompt)
     print("Result (generate):", result_sync)
 
-    # 2. __call__
-    print("\n=== Synchronous __call__ ===")
-    result_call = chatterer(prompt)
-    print("Result (__call__):", result_call)
-
-    # 3. generate_stream
+    # generate_stream
     print("\n=== Synchronous generate_stream ===")
     for i, chunk in enumerate(chatterer.generate_stream(prompt)):
         print(f"Chunk {i}:", chunk)
 
-    # 4. generate_pydantic
+    # generate_pydantic
     print("\n=== Synchronous generate_pydantic ===")
-    try:
-        result_pydantic = chatterer.generate_pydantic(Propositions, prompt)
-        print("Result (generate_pydantic):", result_pydantic)
-    except Exception as e:
-        print("Error in generate_pydantic:", e)
+    result_pydantic = chatterer(prompt, Propositions)
+    print("Result (generate_pydantic):", result_pydantic)
 
-    # 5. generate_pydantic_stream
+    # generate_pydantic_stream
     print("\n=== Synchronous generate_pydantic_stream ===")
-    try:
-        for i, chunk in enumerate(chatterer.generate_pydantic_stream(Propositions, prompt)):
-            print(f"Pydantic Chunk {i}:", chunk)
-    except Exception as e:
-        print("Error in generate_pydantic_stream:", e)
+    for i, chunk in enumerate(chatterer.generate_pydantic_stream(Propositions, prompt)):
+        print(f"Pydantic Chunk {i}:", chunk)
 
     # === Asynchronous Tests ===
 

chatterer/messages.py

@@ -0,0 +1,8 @@
+from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, SystemMessage
+
+__all__ = [
+    "AIMessage",
+    "BaseMessage",
+    "HumanMessage",
+    "SystemMessage",
+]

chatterer/strategies/atom_of_thoughts.py

@@ -6,11 +6,10 @@ from dataclasses import dataclass, field
 from enum import StrEnum
 from typing import Optional, Type, TypeVar
 
-from langchain_core.messages import AIMessage, BaseMessage, HumanMessage
 from pydantic import BaseModel, Field, ValidationError
 
-# Import your Chatterer interface (do not remove)
 from ..language_model import Chatterer, LanguageModelInput
+from ..messages import AIMessage, BaseMessage, HumanMessage
 from .base import BaseStrategy
 
 # ---------------------------------------------------------------------------------

chatterer/tools/__init__.py

@@ -1,3 +1,4 @@
+from .citation_chunking import citation_chunker
 from .convert_to_text import (
     anything_to_markdown,
     get_default_html_to_markdown_options,
@@ -12,4 +13,5 @@ __all__ = [
     "pdf_to_text",
     "get_default_html_to_markdown_options",
     "pyscripts_to_snippets",
+    "citation_chunker",
 ]

chatterer/tools/citation_chunking/__init__.py

@@ -0,0 +1,3 @@
+from .citation_chunker import citation_chunker
+
+__all__ = ["citation_chunker"]

chatterer/tools/citation_chunking/chunks.py

@@ -0,0 +1,53 @@
+import logging
+from typing import Callable, Optional, Self
+
+from pydantic import BaseModel, Field
+
+from ...language_model import Chatterer
+from ...messages import AIMessage, BaseMessage, HumanMessage
+from .prompt import (
+    generate_fewshot_affirmative_response,
+    generate_human_assistant_fewshot_examples,
+    generate_instruction,
+)
+from .reference import Reference
+
+logger = logging.getLogger(__name__)
+
+
+class CitationChunk(BaseModel):
+    subject: str = Field(description="The main topic or subject that the citations capture.")
+    references: list[Reference] = Field(description="A list of citation objects and/or regex patterns for the subject.")
+
+
+class CitationChunks(BaseModel):
+    citation_chunks: list[CitationChunk] = Field(
+        description="A list of citation chunks, each capturing a specific topic in the document."
+    )
+
+    @classmethod
+    def from_llm(
+        cls,
+        chatterer: Chatterer,
+        document: str,
+        fewshot_examples_generator: Optional[
+            Callable[[], list[tuple[str, str]]]
+        ] = generate_human_assistant_fewshot_examples,
+        instruction_generator: Optional[Callable[[], str]] = generate_instruction,
+        fewshot_affirmative_response: Optional[Callable[[], str]] = generate_fewshot_affirmative_response,
+    ) -> Self:
+        messages: list[BaseMessage] = []
+        if instruction_generator:
+            messages.append(HumanMessage(content=instruction_generator()))
+        if fewshot_examples_generator is not None:
+            if fewshot_affirmative_response:
+                messages.append(AIMessage(content=generate_fewshot_affirmative_response()))
+            for human_ask, ai_answer in fewshot_examples_generator():
+                messages.append(HumanMessage(content=human_ask))
+                messages.append(AIMessage(content=ai_answer))
+        messages.append(HumanMessage(content=document))
+        try:
+            return chatterer.generate_pydantic(response_model=cls, messages=messages)
+        except Exception as e:
+            logger.error(f"Error obtaining CitationChunks from LLM: {e}")
+            raise e

chatterer/tools/citation_chunking/citation_chunker.py

@@ -0,0 +1,118 @@
+import logging
+from typing import Callable, NamedTuple, Optional, Self
+
+import colorama
+from colorama import Fore
+
+from ...language_model import Chatterer
+from .chunks import CitationChunks
+from .citations import Citations
+from .prompt import (
+    generate_fewshot_affirmative_response,
+    generate_human_assistant_fewshot_examples,
+    generate_instruction,
+)
+
+logger = logging.getLogger(__name__)
+colorama.init()
+
+
+class GlobalCoverage(NamedTuple):
+    coverage: float
+    matched_intervals: list[tuple[int, int]]
+
+    @staticmethod
+    def merge_intervals(intervals: list[tuple[int, int]]) -> list[tuple[int, int]]:
+        if not intervals:
+            return []
+        sorted_intervals = sorted(intervals, key=lambda x: x[0])
+        merged: list[tuple[int, int]] = [sorted_intervals[0]]
+        for current in sorted_intervals[1:]:
+            prev = merged[-1]
+            if current[0] <= prev[1]:
+                merged[-1] = (prev[0], max(prev[1], current[1]))
+            else:
+                merged.append(current)
+        return merged
+
+    @classmethod
+    def from_verified_citations(cls, verified_chunks: list[Citations], document: str) -> Self:
+        all_intervals: list[tuple[int, int]] = []
+        for chunk in verified_chunks:
+            for matches in chunk.references.values():
+                for m in matches:
+                    all_intervals.append((m.start_idx, m.end_idx))
+        merged: list[tuple[int, int]] = cls.merge_intervals(all_intervals)
+        doc_length: int = len(document)
+        total_matched = sum((e - s for s, e in merged))
+        coverage: float = total_matched / doc_length if doc_length > 0 else 0.0
+        return cls(coverage=coverage, matched_intervals=merged)
+
+
+def citation_chunker(
+    document: str,
+    chatterer: Chatterer,
+    global_coverage_threshold: float = 0.9,
+    num_refinement_steps: int = 3,
+    fewshot_examples_generator: Optional[
+        Callable[[], list[tuple[str, str]]]
+    ] = generate_human_assistant_fewshot_examples,
+    instruction_generator: Optional[Callable[[], str]] = generate_instruction,
+    fewshot_affirmative_response: Optional[Callable[[], str]] = generate_fewshot_affirmative_response,
+    test_global_coverage: bool = False,
+) -> list[Citations]:
+    """
+    1) Obtain CitationChunks via the LLM.
+    2) Process each chunk to extract MatchedText using snippet-based index correction.
+    3) Calculate overall document coverage and print results.
+    """
+    unverified_chunks: CitationChunks = CitationChunks.from_llm(
+        chatterer=chatterer,
+        document=document,
+        fewshot_examples_generator=fewshot_examples_generator,
+        instruction_generator=instruction_generator,
+        fewshot_affirmative_response=fewshot_affirmative_response,
+    )
+
+    verified_chunks: list[Citations] = []
+    for chunk in unverified_chunks.citation_chunks:
+        try:
+            vc: Citations = Citations.from_unverified(
+                unverified_chunk=chunk,
+                document=document,
+                model_and_refinement_steps=(chatterer, num_refinement_steps),
+            )
+            verified_chunks.append(vc)
+        except Exception as e:
+            logger.error(f"Error processing chunk for subject '{chunk.subject}': {e}")
+
+    if test_global_coverage:
+        gc = GlobalCoverage.from_verified_citations(verified_chunks, document)
+        logger.info(f"Global coverage: {gc.coverage * 100:.1f}%")
+        if gc.coverage < global_coverage_threshold:
+            logger.info(
+                f"Global coverage {gc.coverage * 100:.1f}% is below the threshold {global_coverage_threshold * 100:.1f}%."
+            )
+        print("=== Final Global Coverage Check ===")
+        print(f"Overall coverage: {gc.coverage * 100:.1f}% of the document.")
+        if gc.matched_intervals:
+            print("Merged matched intervals:")
+            for interval in gc.matched_intervals:
+                print(f" - {interval}")
+        else:
+            print("No matches found across all chunks.")
+        print("\n=== Raw Semantic Chunking Result ===")
+        for vc in verified_chunks:
+            print(f"{Fore.LIGHTGREEN_EX}[SUBJECT] {Fore.GREEN}{vc.name}{Fore.RESET}")
+            if vc.references:
+                for source_key, matches in vc.references.items():
+                    print(f"{Fore.LIGHTBLUE_EX}  [SOURCE] {Fore.BLUE}{source_key}{Fore.RESET}")
+                    for mt in matches:
+                        snippet = repr(mt.text)
+                        print(
+                            f"    {Fore.LIGHTYELLOW_EX}[MATCH @ {mt.start_idx}~{mt.end_idx}] {Fore.YELLOW}{snippet}{Fore.RESET}"
+                        )
+            else:
+                print(" - (No matches found even after refinement.)")
+
+    return verified_chunks

chatterer/tools/citation_chunking/citations.py

@@ -0,0 +1,285 @@
+from __future__ import annotations
+
+import difflib
+import logging
+from typing import NamedTuple, Optional, Self, TypeAlias
+
+from pydantic import Field
+from regex import DOTALL
+from regex import compile as regex_compile
+from regex import error as regex_error
+
+from ...language_model import Chatterer
+from ...messages import HumanMessage
+from .chunks import CitationChunk
+from .reference import MultiMatchRegex, Reference, SingleMatchCitation
+from .utils import MatchedText
+
+ModelAndSteps: TypeAlias = tuple[Chatterer, int]
+logger = logging.getLogger(__name__)
+
+
+class Citations(NamedTuple):
+    """
+    Holds the verified citation chunks and their matching information.
+    """
+
+    name: str
+    references: dict[Reference, list[ReferencedTextMatch]]
+
+    @classmethod
+    def from_unverified(
+        cls,
+        unverified_chunk: CitationChunk,
+        document: str,
+        model_and_refinement_steps: Optional[ModelAndSteps] = None,  # Optional LLM for refinement
+    ) -> Self:
+        subject: str = unverified_chunk.subject
+        self: Self = cls(name=subject, references={})
+        for reference in unverified_chunk.references or ():
+            if isinstance(reference, SingleMatchCitation):
+                try:
+                    mt: Optional[ReferencedTextMatch] = ReferencedTextMatch.from_citation(
+                        subject=subject,
+                        citation=reference,
+                        document=document,
+                        model_and_refinement_steps=model_and_refinement_steps,
+                    )
+                    if mt is None or not mt.text.strip():
+                        logger.warning(f"Failed to extract text for citation {reference} in subject '{subject}'.")
+                    else:
+                        self.references[reference] = [mt]
+                except Exception as e:
+                    logger.error(f"Error processing citation {reference} for subject '{subject}': {e}")
+            else:
+                try:
+                    regex_matches: list[ReferencedTextMatch] = ReferencedTextMatch.from_regex(
+                        regex=reference, subject=subject, document=document
+                    )
+                    if regex_matches:
+                        self.references[reference] = regex_matches
+                except regex_error as e:
+                    logger.error(f"Regex error for subject '{subject}' with pattern '{reference}': {e}")
+        return self
+
+
+class ReferencedTextMatch(MatchedText):
+    @classmethod
+    def from_citation(
+        cls,
+        subject: str,
+        citation: SingleMatchCitation,
+        document: str,
+        model_and_refinement_steps: Optional[ModelAndSteps] = None,  # Optional LLM for quality-check refinement
+    ) -> Optional[Self]:
+        """
+        Extract text from the document using the adjusted citation indices.
+        Additionally, if a language model is provided, evaluate the extraction quality
+        and refine it if needed.
+        """
+        citation_id: Optional[SingleMatchCitationWithIndex] = SingleMatchCitationWithIndex.from_indexless_citation(
+            indexless_citation=citation,
+            document=document,
+            subject=subject,
+            model_and_refinement_steps=model_and_refinement_steps,
+        )
+        if citation_id is None:
+            return
+
+        return cls(
+            start_idx=citation_id.start,
+            end_idx=citation_id.end,
+            text=citation_id.extracted_text,
+        )
+
+    @classmethod
+    def from_regex(cls, regex: MultiMatchRegex, subject: str, document: str) -> list[Self]:
+        """
+        Apply the given regex to the document and return all matching results as a list of MatchedText.
+        """
+        try:
+            compiled_pattern = regex_compile(regex.regular_expression, flags=DOTALL)
+        except regex_error as e:
+            logger.error(f"Regex compilation error for pattern /{regex.regular_expression}/: {e}")
+            raise e
+        try:
+            matches = list(compiled_pattern.finditer(document, timeout=1.0))
+        except regex_error as e:
+            logger.error(f"Regex matching error for pattern /{regex.regular_expression}/: {e}")
+            raise e
+        return [cls(start_idx=m.start(), end_idx=m.end(), text=m.group()) for m in matches]
+
+
+class SingleMatchCitationWithIndex(SingleMatchCitation):
+    start: int = Field(description="The computed start index of the citation in the document.")
+    end: int = Field(description="The computed end index of the citation in the document.")
+    extracted_text: str = Field(description="The extracted text from the document using the computed indices.")
+
+    @classmethod
+    def from_indexless_citation(
+        cls,
+        indexless_citation: SingleMatchCitation,
+        document: str,
+        subject: str,
+        model_and_refinement_steps: Optional[ModelAndSteps] = None,  # Optional LLM for quality-check refinement
+    ) -> Optional[Self]:
+        """
+        Compute the correct start and end indices for the citation based on the provided text snippets.
+        This method ignores any indices provided by the LLM and computes them using a similarity-based search.
+        If multiple high-scoring candidates are found, the one with the highest effective score is chosen.
+        """
+        if model_and_refinement_steps is None:
+            model = None
+            num_refinement_steps = 1
+        else:
+            model, num_refinement_steps = model_and_refinement_steps
+        for _ in range(num_refinement_steps):
+            result = cls.from_indexless_citation_with_refinement(
+                indexless_citation=indexless_citation,
+                document=document,
+                subject=subject,
+                chatterer=model,
+            )
+            if result is None:
+                continue
+            return result
+
+    @staticmethod
+    def find_best_match_index(snippet: str, document: str, target_index: int) -> Optional[int]:
+        """
+        Extracts a candidate window centered around the specified target_index,
+        with a size equal to the length of the snippet. Within this region,
+        it calculates the similarity with the snippet using a sliding window approach.
+
+        The index of the candidate with the highest effective_score is returned.
+        If no suitable candidate is found, the target_index is returned.
+
+        Note: If multiple high-scoring candidates are found, the one with the highest effective score is chosen.
+        """
+        snippet = snippet.strip()
+        if not snippet:
+            return
+        snippet_len: int = len(snippet)
+        best_index: int = -1
+        best_effective_score = 0.0
+        max_radius = max(target_index, len(document) - target_index)
+        for offset in range(max_radius):
+            for candidate_index in (
+                target_index - offset,
+                target_index + offset,
+            ):
+                if candidate_index < 0 or candidate_index + snippet_len > len(document):
+                    continue
+                candidate_segment = document[candidate_index : min(candidate_index + snippet_len, len(document))]
+                if len(candidate_segment) < snippet_len:
+                    continue
+                local_best_similarity = 0.0
+                local_best_offset = 0
+                for i in range(0, len(candidate_segment) - snippet_len + 1):
+                    candidate_window = candidate_segment[i : i + snippet_len]
+                    similarity = difflib.SequenceMatcher(None, snippet, candidate_window).ratio()
+                    if similarity > local_best_similarity:
+                        local_best_similarity = similarity
+                        local_best_offset = i
+                candidate_final_index = candidate_index + local_best_offset
+                if candidate_final_index + snippet_len > len(document):
+                    candidate_final_index = len(document) - snippet_len
+                if local_best_similarity > best_effective_score:
+                    best_effective_score = local_best_similarity
+                    best_index = candidate_final_index
+        if not 0 <= best_index < len(document):
+            logger.warning(f"Snippet '{snippet}' not found with sufficient similarity.")
+            return
+        else:
+            logger.debug(
+                f"Found best match for snippet '{snippet}' at index {best_index} with effective score {best_effective_score:.2f}."
+            )
+            return best_index
+
+    @classmethod
+    def from_indexless_citation_with_refinement(
+        cls,
+        indexless_citation: SingleMatchCitation,
+        document: str,
+        subject: str,
+        chatterer: Optional[Chatterer],
+    ) -> Optional[Self]:
+        if chatterer is None:
+            logger.error("No LLM provided for indexless citation refinement.")
+            new_indexless_citation = indexless_citation
+        else:
+            new_indexless_citation = chatterer.generate_pydantic(
+                response_model=SingleMatchCitation,
+                messages=[
+                    HumanMessage(
+                        content=(
+                            "I tried to find the `SNIPPET` in the `original-raw-document` to extract a text citation for the subject `subject-to-parse`, but I couldn't find it. "
+                            "Please provide `citation-start-from` and `citation-end-at` to help me locate the correct text span.\n"
+                            "---\n"
+                            "<original-raw-document>\n"
+                            f"{document}\n"
+                            "</original-raw-document>\n"
+                            "---\n"
+                            "<subject-to-parse>\n"
+                            f"{subject}\n"
+                            "</subject-to-parse>\n"
+                            "---\n"
+                            "<current-citation-start-from>\n"
+                            f"{indexless_citation.start_from}\n"
+                            "</current-citation-start-from>\n"
+                            "---\n"
+                            "<current-citation-end-at>\n"
+                            f"{indexless_citation.end_at}\n"
+                            "</current-citation-end-at>\n"
+                        )
+                    ),
+                ],
+            )
+        doc_len: int = len(document)
+
+        start_snippet: str = new_indexless_citation.start_from.strip()
+        if start_snippet:
+            target_for_start = document.find(start_snippet)
+            if target_for_start == -1:
+                target_for_start = 0
+            new_start: Optional[int] = cls.find_best_match_index(
+                snippet=start_snippet,
+                document=document,
+                target_index=target_for_start,
+            )
+            if new_start is None:
+                return
+        else:
+            logger.warning("No start_text provided")
+            return
+        end_snippet: str = new_indexless_citation.end_at.strip()
+        if end_snippet:
+            target_for_end = document.find(end_snippet, new_start)
+            if target_for_end == -1:
+                target_for_end = new_start
+            candidate_end: Optional[int] = cls.find_best_match_index(
+                snippet=end_snippet,
+                document=document,
+                target_index=target_for_end,
+            )
+            if candidate_end is None:
+                return
+            new_end: int = candidate_end + len(end_snippet)
+        else:
+            logger.warning("No end_text provided; defaulting end index to document length.")
+            new_end = doc_len
+        if not 0 <= new_start < new_end <= doc_len:
+            logger.error(f"Adjusted citation indices invalid: start {new_start}, end {new_end}, doc_len {doc_len}.")
+            return
+        try:
+            extracted_text = document[new_start:new_end]
+        except IndexError as e:
+            logger.error(f"Error extracting text using adjusted citation indices: {e}")
+            return
+        return cls(
+            start=new_start,
+            end=new_end,
+            start_from=new_indexless_citation.start_from,
+            end_at=new_indexless_citation.end_at,
+            extracted_text=extracted_text,
+        )