lexoid 0.1.13__py3-none-any.whl → 0.1.15__py3-none-any.whl

lexoid/api.py

@@ -4,13 +4,19 @@ import re
 import tempfile
 from concurrent.futures import ProcessPoolExecutor
 from enum import Enum
+from functools import wraps
 from glob import glob
 from time import time
-from typing import Union, Dict, List
+from typing import Optional, Union, Dict, List
 
 from loguru import logger
 
-from lexoid.core.parse_type.llm_parser import parse_llm_doc
+from lexoid.core.parse_type.llm_parser import (
+    parse_llm_doc,
+    create_response,
+    convert_doc_to_base64_images,
+    get_api_provider_for_model,
+)
 from lexoid.core.parse_type.static_parser import parse_static_doc
 from lexoid.core.utils import (
     convert_to_pdf,
@@ -31,6 +37,51 @@ class ParserType(Enum):
     AUTO = "AUTO"
 
 
+def retry_with_different_parser_type(func):
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        try:
+            if len(args) > 0:
+                kwargs["path"] = args[0]
+            if len(args) > 1:
+                router_priority = kwargs.get("router_priority", "speed")
+                if args[1] == ParserType.AUTO:
+                    parser_type = ParserType[router(kwargs["path"], router_priority)]
+                    logger.debug(f"Auto-detected parser type: {parser_type}")
+                    kwargs["routed"] = True
+                else:
+                    parser_type = args[1]
+                kwargs["parser_type"] = parser_type
+            return func(**kwargs)
+        except Exception as e:
+            if kwargs.get("parser_type") == ParserType.LLM_PARSE and kwargs.get(
+                "routed", False
+            ):
+                logger.warning(
+                    f"LLM_PARSE failed with error: {e}. Retrying with STATIC_PARSE."
+                )
+                kwargs["parser_type"] = ParserType.STATIC_PARSE
+                kwargs["routed"] = False
+                return func(**kwargs)
+            elif kwargs.get("parser_type") == ParserType.STATIC_PARSE and kwargs.get(
+                "routed", False
+            ):
+                logger.warning(
+                    f"STATIC_PARSE failed with error: {e}. Retrying with LLM_PARSE."
+                )
+                kwargs["parser_type"] = ParserType.LLM_PARSE
+                kwargs["routed"] = False
+                return func(**kwargs)
+            else:
+                logger.error(
+                    f"Parsing failed with error: {e}. No fallback parser available."
+                )
+                raise e
+
+    return wrapper
+
+
+@retry_with_different_parser_type
 def parse_chunk(path: str, parser_type: ParserType, **kwargs) -> Dict:
     """
     Parses a file using the specified parser type.
@@ -49,21 +100,20 @@ def parse_chunk(path: str, parser_type: ParserType, **kwargs) -> Dict:
             - parent_title: Title of parent doc if recursively parsed
             - recursive_docs: List of dictionaries for recursively parsed documents
             - token_usage: Dictionary containing token usage statistics
+            - parser_used: Which parser was actually used
     """
-    if parser_type == ParserType.AUTO:
-        router_priority = kwargs.get("router_priority", "speed")
-        parser_type = ParserType[router(path, router_priority)]
-        logger.debug(f"Auto-detected parser type: {parser_type}")
-
     kwargs["start"] = (
         int(os.path.basename(path).split("_")[1]) - 1 if kwargs.get("split") else 0
     )
     if parser_type == ParserType.STATIC_PARSE:
         logger.debug("Using static parser")
-        return parse_static_doc(path, **kwargs)
+        result = parse_static_doc(path, **kwargs)
     else:
         logger.debug("Using LLM parser")
-        return parse_llm_doc(path, **kwargs)
+        result = parse_llm_doc(path, **kwargs)
+
+    result["parser_used"] = parser_type
+    return result
 
 
 def parse_chunk_list(
@@ -82,15 +132,18 @@ def parse_chunk_list(
     """
     combined_segments = []
     raw_texts = []
-    token_usage = {"input": 0, "output": 0, "image_count": 0}
+    token_usage = {"input": 0, "output": 0, "llm_page_count": 0}
     for file_path in file_paths:
         result = parse_chunk(file_path, parser_type, **kwargs)
         combined_segments.extend(result["segments"])
         raw_texts.append(result["raw"])
-        if "token_usage" in result:
+        if (
+            result.get("parser_used") == ParserType.LLM_PARSE
+            and "token_usage" in result
+        ):
             token_usage["input"] += result["token_usage"]["input"]
             token_usage["output"] += result["token_usage"]["output"]
-            token_usage["image_count"] += len(result["segments"])
+            token_usage["llm_page_count"] += len(result["segments"])
     token_usage["total"] = token_usage["input"] + token_usage["output"]
 
     return {
@@ -136,7 +189,7 @@ def parse(
     as_pdf = kwargs.get("as_pdf", False)
     depth = kwargs.get("depth", 1)
 
-    if type(parser_type) == str:
+    if type(parser_type) is str:
         parser_type = ParserType[parser_type]
     if (
         path.lower().endswith((".doc", ".docx"))
@@ -182,9 +235,9 @@ def parse(
             sub_pdf_path = os.path.join(sub_pdf_dir, f"{os.path.basename(path)}")
             path = create_sub_pdf(path, sub_pdf_path, kwargs["page_nums"])
 
-        if not path.lower().endswith(".pdf") or parser_type == ParserType.STATIC_PARSE:
+        if not path.lower().endswith(".pdf"):
             kwargs["split"] = False
-            result = parse_chunk(path, parser_type, **kwargs)
+            result = parse_chunk_list([path], parser_type, kwargs)
         else:
             kwargs["split"] = True
             split_dir = os.path.join(temp_dir, "splits/")
@@ -219,42 +272,43 @@ def parse(
                 "token_usage": {
                     "input": sum(r["token_usage"]["input"] for r in chunk_results),
                     "output": sum(r["token_usage"]["output"] for r in chunk_results),
-                    "image_count": sum(
-                        r["token_usage"]["image_count"] for r in chunk_results
+                    "llm_page_count": sum(
+                        r["token_usage"]["llm_page_count"] for r in chunk_results
                     ),
                     "total": sum(r["token_usage"]["total"] for r in chunk_results),
                 },
             }
 
-            if "api_cost_mapping" in kwargs:
-                api_cost_mapping = kwargs["api_cost_mapping"]
-                if isinstance(api_cost_mapping, dict):
-                    api_cost_mapping = api_cost_mapping
-                elif isinstance(api_cost_mapping, str) and os.path.exists(
-                    api_cost_mapping
-                ):
-                    with open(api_cost_mapping, "r") as f:
-                        api_cost_mapping = json.load(f)
-                else:
-                    raise ValueError(f"Unsupported API cost value: {api_cost_mapping}.")
+        if "api_cost_mapping" in kwargs and "token_usage" in result:
+            api_cost_mapping = kwargs["api_cost_mapping"]
+            if isinstance(api_cost_mapping, dict):
+                api_cost_mapping = api_cost_mapping
+            elif isinstance(api_cost_mapping, str) and os.path.exists(api_cost_mapping):
+                with open(api_cost_mapping, "r") as f:
+                    api_cost_mapping = json.load(f)
+            else:
+                raise ValueError(f"Unsupported API cost value: {api_cost_mapping}.")
 
-                api_cost = api_cost_mapping.get(
-                    kwargs.get("model", "gemini-2.0-flash"), None
+            api_cost = api_cost_mapping.get(
+                kwargs.get("model", "gemini-2.0-flash"), None
+            )
+            if api_cost:
+                token_usage = result["token_usage"]
+                token_cost = {
+                    "input": token_usage["input"] * api_cost["input"] / 1_000_000,
+                    "input-image": api_cost.get("input-image", 0)
+                    * token_usage.get("llm_page_count", 0),
+                    "output": token_usage["output"] * api_cost["output"] / 1_000_000,
+                }
+                token_cost["total"] = (
+                    token_cost["input"]
+                    + token_cost["input-image"]
+                    + token_cost["output"]
                 )
-                if api_cost:
-                    token_usage = result["token_usage"]
-                    token_cost = {
-                        "input": token_usage["input"] * api_cost["input"] / 1_000_000
-                        + api_cost.get("input-image", 0) * token_usage["image_count"],
-                        "output": token_usage["output"]
-                        * api_cost["output"]
-                        / 1_000_000,
-                    }
-                    token_cost["total"] = token_cost["input"] + token_cost["output"]
-                    result["token_cost"] = token_cost
-
-            if as_pdf:
-                result["pdf_path"] = path
+                result["token_cost"] = token_cost
+
+        if as_pdf:
+            result["pdf_path"] = path
 
     if depth > 1:
         recursive_docs = []
@@ -285,3 +339,71 @@ def parse(
         result["recursive_docs"] = recursive_docs
 
     return result
+
+
+def parse_with_schema(
+    path: str,
+    schema: Dict,
+    api: Optional[str] = None,
+    model: str = "gpt-4o-mini",
+    **kwargs,
+) -> List[List[Dict]]:
+    """
+    Parses a PDF using an LLM to generate structured output conforming to a given JSON schema.
+
+    Args:
+        path (str): Path to the PDF file.
+        schema (Dict): JSON schema to which the parsed output should conform.
+        api (str, optional): LLM API provider (One of "openai", "huggingface", "together", "openrouter", and "fireworks").
+        model (str, optional): LLM model name.
+        **kwargs: Additional arguments for the parser (e.g.: temperature, max_tokens).
+
+    Returns:
+        List[List[Dict]]: List of dictionaries for each page, each conforming to the provided schema.
+    """
+    if not api:
+        api = get_api_provider_for_model(model)
+        logger.debug(f"Using API provider: {api}")
+
+    system_prompt = f"""
+        The output should be formatted as a JSON instance that conforms to the JSON schema below.
+
+        As an example, for the schema {{
+        "properties": {{
+            "foo": {{
+            "title": "Foo",
+            "description": "a list of strings",
+            "type": "array",
+            "items": {{"type": "string"}}
+            }}
+        }},
+        "required": ["foo"]
+        }}, the object {{"foo": ["bar", "baz"]}} is valid. The object {{"properties": {{"foo": ["bar", "baz"]}}}} is not.
+
+        Here is the output schema:
+        {json.dumps(schema, indent=2)}
+
+        """
+
+    user_prompt = "You are an AI agent that parses documents and returns them in the specified JSON format. Please parse the document and return it in the required format."
+
+    responses = []
+    images = convert_doc_to_base64_images(path)
+    for i, (page_num, image) in enumerate(images):
+        resp_dict = create_response(
+            api=api,
+            model=model,
+            user_prompt=user_prompt,
+            system_prompt=system_prompt,
+            image_url=image,
+            temperature=kwargs.get("temperature", 0.0),
+            max_tokens=kwargs.get("max_tokens", 1024),
+        )
+
+        response = resp_dict.get("response", "")
+        response = response.split("```json")[-1].split("```")[0].strip()
+        logger.debug(f"Processing page {page_num + 1} with response: {response}")
+        new_dict = json.loads(response)
+        responses.append(new_dict)
+
+    return responses

lexoid/core/parse_type/llm_parser.py

@@ -3,23 +3,25 @@ import io
 import mimetypes
 import os
 import time
+from functools import wraps
+from typing import Dict, List, Optional, Tuple
+
 import pypdfium2 as pdfium
 import requests
-from functools import wraps
+from anthropic import Anthropic
+from huggingface_hub import InferenceClient
+from loguru import logger
+from openai import OpenAI
 from requests.exceptions import HTTPError
-from typing import Dict, List
+from together import Together
 
 from lexoid.core.prompt_templates import (
     INSTRUCTIONS_ADD_PG_BREAK,
+    LLAMA_PARSER_PROMPT,
     OPENAI_USER_PROMPT,
     PARSER_PROMPT,
-    LLAMA_PARSER_PROMPT,
 )
 from lexoid.core.utils import convert_image_to_pdf
-from loguru import logger
-from openai import OpenAI
-from together import Together
-from huggingface_hub import InferenceClient
 
 
 def retry_on_http_error(func):
@@ -48,33 +50,41 @@ def retry_on_http_error(func):
     return wrapper
 
 
-@retry_on_http_error
-def parse_llm_doc(path: str, **kwargs) -> List[Dict] | str:
-    if "api_provider" in kwargs and kwargs["api_provider"]:
-        return parse_with_api(path, api=kwargs["api_provider"], **kwargs)
-    if "model" not in kwargs:
-        kwargs["model"] = "gemini-2.0-flash"
-    model = kwargs.get("model")
+def get_api_provider_for_model(model: str) -> str:
     if model.startswith("gemini"):
-        return parse_with_gemini(path, **kwargs)
+        return "gemini"
     if model.startswith("gpt"):
-        return parse_with_api(path, api="openai", **kwargs)
+        return "openai"
     if model.startswith("meta-llama"):
         if "Turbo" in model or model == "meta-llama/Llama-Vision-Free":
-            return parse_with_api(path, api="together", **kwargs)
-        return parse_with_api(path, api="huggingface", **kwargs)
+            return "together"
+        return "huggingface"
     if any(model.startswith(prefix) for prefix in ["microsoft", "google", "qwen"]):
-        return parse_with_api(path, api="openrouter", **kwargs)
+        return "openrouter"
+    if model.startswith("accounts/fireworks"):
+        return "fireworks"
+    if model.startswith("claude"):
+        return "anthropic"
     raise ValueError(f"Unsupported model: {model}")
 
 
-def parse_with_gemini(path: str, **kwargs) -> List[Dict] | str:
-    api_key = os.environ.get("GOOGLE_API_KEY")
-    if not api_key:
-        raise ValueError("GOOGLE_API_KEY environment variable is not set")
+@retry_on_http_error
+def parse_llm_doc(path: str, **kwargs) -> List[Dict] | str:
+    if "api_provider" in kwargs and kwargs["api_provider"]:
+        return parse_with_api(path, api=kwargs["api_provider"], **kwargs)
 
-    url = f"https://generativelanguage.googleapis.com/v1beta/models/{kwargs['model']}:generateContent?key={api_key}"
+    model = kwargs.get("model", "gemini-2.0-flash")
+    kwargs["model"] = model
+
+    api_provider = get_api_provider_for_model(model)
 
+    if api_provider == "gemini":
+        return parse_with_gemini(path, **kwargs)
+    else:
+        return parse_with_api(path, api=api_provider, **kwargs)
+
+
+def parse_with_gemini(path: str, **kwargs) -> List[Dict] | str:
     # Check if the file is an image and convert to PDF if necessary
     mime_type, _ = mimetypes.guess_type(path)
     if mime_type and mime_type.startswith("image"):
@@ -86,6 +96,20 @@ def parse_with_gemini(path: str, **kwargs) -> List[Dict] | str:
             file_content = file.read()
         base64_file = base64.b64encode(file_content).decode("utf-8")
 
+    return parse_image_with_gemini(
+        base64_file=base64_file, mime_type=mime_type, **kwargs
+    )
+
+
+def parse_image_with_gemini(
+    base64_file: str, mime_type: str = "image/png", **kwargs
+) -> List[Dict] | str:
+    api_key = os.environ.get("GOOGLE_API_KEY")
+    if not api_key:
+        raise ValueError("GOOGLE_API_KEY environment variable is not set")
+
+    url = f"https://generativelanguage.googleapis.com/v1beta/models/{kwargs['model']}:generateContent?key={api_key}"
+
     if "system_prompt" in kwargs:
         prompt = kwargs["system_prompt"]
     else:
@@ -105,7 +129,7 @@ def parse_with_gemini(path: str, **kwargs) -> List[Dict] | str:
             }
         ],
         "generationConfig": {
-            "temperature": kwargs.get("temperature", 0.7),
+            "temperature": kwargs.get("temperature", 0),
         },
     }
 
@@ -125,24 +149,23 @@ def parse_with_gemini(path: str, **kwargs) -> List[Dict] | str:
         if "text" in part
     )
 
-    combined_text = ""
+    combined_text = raw_text
     if "<output>" in raw_text:
-        combined_text = raw_text.split("<output>")[1].strip()
-    if "</output>" in result:
-        combined_text = result.split("</output>")[0].strip()
+        combined_text = raw_text.split("<output>")[-1].strip()
+    if "</output>" in combined_text:
+        combined_text = combined_text.split("</output>")[0].strip()
 
     token_usage = result["usageMetadata"]
     input_tokens = token_usage.get("promptTokenCount", 0)
     output_tokens = token_usage.get("candidatesTokenCount", 0)
     total_tokens = input_tokens + output_tokens
-
     return {
         "raw": combined_text.replace("<page-break>", "\n\n"),
         "segments": [
             {"metadata": {"page": kwargs.get("start", 0) + page_no}, "content": page}
             for page_no, page in enumerate(combined_text.split("<page-break>"), start=1)
         ],
-        "title": kwargs["title"],
+        "title": kwargs.get("title", ""),
         "url": kwargs.get("url", ""),
         "parent_title": kwargs.get("parent_title", ""),
         "recursive_docs": [],
@@ -169,18 +192,54 @@ def convert_pdf_page_to_base64(
     return base64.b64encode(img_byte_arr.getvalue()).decode("utf-8")
 
 
-def parse_with_api(path: str, api: str, **kwargs) -> List[Dict] | str:
-    """
-    Parse documents (PDFs or images) using various vision model APIs.
+def get_messages(
+    system_prompt: Optional[str], user_prompt: Optional[str], image_url: Optional[str]
+) -> List[Dict]:
+    messages = []
+    if system_prompt:
+        messages.append(
+            {
+                "role": "system",
+                "content": system_prompt,
+            }
+        )
+    base_message = (
+        [
+            {"type": "text", "text": user_prompt},
+        ]
+        if user_prompt
+        else []
+    )
+    image_message = (
+        [
+            {
+                "type": "image_url",
+                "image_url": {"url": image_url},
+            }
+        ]
+        if image_url
+        else []
+    )
 
-    Args:
-        path (str): Path to the document to parse
-        api (str): Which API to use ("openai", "huggingface", or "together")
-        **kwargs: Additional arguments including model, temperature, title, etc.
+    messages.append(
+        {
+            "role": "user",
+            "content": base_message + image_message,
+        }
+    )
 
-    Returns:
-        Dict: Dictionary containing parsed document data
-    """
+    return messages
+
+
+def create_response(
+    api: str,
+    model: str,
+    system_prompt: Optional[str] = None,
+    user_prompt: Optional[str] = None,
+    image_url: Optional[str] = None,
+    temperature: float = 0.0,
+    max_tokens: int = 1024,
+) -> Dict:
     # Initialize appropriate client
     clients = {
         "openai": lambda: OpenAI(),
@@ -192,11 +251,110 @@ def parse_with_api(path: str, api: str, **kwargs) -> List[Dict] | str:
             base_url="https://openrouter.ai/api/v1",
             api_key=os.environ["OPENROUTER_API_KEY"],
         ),
+        "fireworks": lambda: OpenAI(
+            base_url="https://api.fireworks.ai/inference/v1",
+            api_key=os.environ["FIREWORKS_API_KEY"],
+        ),
+        "anthropic": lambda: Anthropic(
+            api_key=os.environ["ANTHROPIC_API_KEY"],
+        ),
+        "gemini": lambda: None,  # Gemini is handled separately
     }
     assert api in clients, f"Unsupported API: {api}"
-    logger.debug(f"Parsing with {api} API and model {kwargs['model']}")
+
+    if api == "gemini":
+        image_url = image_url.split("data:image/png;base64,")[1]
+        response = parse_image_with_gemini(
+            base64_file=image_url,
+            model=model,
+            temperature=temperature,
+            max_tokens=max_tokens,
+            system_prompt=system_prompt,
+        )
+        return {
+            "response": response["raw"],
+            "usage": response["token_usage"],
+        }
+
     client = clients[api]()
 
+    if api == "anthropic":
+        image_media_type = image_url.split(";")[0].split(":")[1]
+        image_data = image_url.split(",")[1]
+        response = client.messages.create(
+            model=model,
+            messages=[
+                {
+                    "role": "user",
+                    "content": [
+                        {
+                            "type": "image",
+                            "source": {
+                                "type": "base64",
+                                "media_type": image_media_type,
+                                "data": image_data,
+                            },
+                        },
+                        {"type": "text", "text": user_prompt},
+                    ],
+                }
+            ],
+            max_tokens=max_tokens,
+            temperature=temperature,
+        )
+
+        return {
+            "response": response.content[0].text,
+            "usage": {
+                "input_tokens": response.usage.input_tokens,
+                "output_tokens": response.usage.output_tokens,
+                "total_tokens": response.usage.input_tokens
+                + response.usage.output_tokens,
+            },
+        }
+
+    # Prepare messages for the API call
+    messages = get_messages(system_prompt, user_prompt, image_url)
+
+    # Common completion parameters
+    completion_params = {
+        "model": model,
+        "messages": messages,
+        "max_tokens": max_tokens,
+        "temperature": temperature,
+    }
+
+    # Get completion from selected API
+    response = client.chat.completions.create(**completion_params)
+    token_usage = response.usage
+
+    # Extract the response text
+    page_text = response.choices[0].message.content
+
+    return {
+        "response": page_text,
+        "usage": {
+            "input_tokens": token_usage.prompt_tokens,
+            "output_tokens": token_usage.completion_tokens,
+            "total_tokens": token_usage.total_tokens,
+        },
+    }
+
+
+def parse_with_api(path: str, api: str, **kwargs) -> List[Dict] | str:
+    """
+    Parse documents (PDFs or images) using various vision model APIs.
+
+    Args:
+        path (str): Path to the document to parse
+        api (str): Which API to use ("openai", "huggingface", or "together")
+        **kwargs: Additional arguments including model, temperature, title, etc.
+
+    Returns:
+        Dict: Dictionary containing parsed document data
+    """
+    logger.debug(f"Parsing with {api} API and model {kwargs['model']}")
+
     # Handle different input types
     mime_type, _ = mimetypes.guess_type(path)
     if mime_type and mime_type.startswith("image"):
@@ -215,76 +373,48 @@ def parse_with_api(path: str, api: str, **kwargs) -> List[Dict] | str:
             for page_num in range(len(pdf_document))
         ]
 
-    # API-specific message formatting
-    def get_messages(page_num: int, image_url: str) -> List[Dict]:
-        image_message = {
-            "type": "image_url",
-            "image_url": {"url": image_url},
-        }
-
+    # Process each page/image
+    all_results = []
+    for page_num, image_url in images:
         if api == "openai":
             system_prompt = kwargs.get(
                 "system_prompt", PARSER_PROMPT.format(custom_instructions="")
             )
             user_prompt = kwargs.get("user_prompt", OPENAI_USER_PROMPT)
-            return [
-                {
-                    "role": "system",
-                    "content": system_prompt,
-                },
-                {
-                    "role": "user",
-                    "content": [
-                        {"type": "text", "text": user_prompt},
-                        image_message,
-                    ],
-                },
-            ]
         else:
-            prompt = kwargs.get("system_prompt", LLAMA_PARSER_PROMPT)
-            base_message = {"type": "text", "text": prompt}
-            return [
-                {
-                    "role": "user",
-                    "content": [base_message, image_message],
-                }
-            ]
-
-    # Process each page/image
-    all_results = []
-    for page_num, image_url in images:
-        messages = get_messages(page_num, image_url)
-
-        # Common completion parameters
-        completion_params = {
-            "model": kwargs["model"],
-            "messages": messages,
-            "max_tokens": kwargs.get("max_tokens", 1024),
-            "temperature": kwargs.get("temperature", 0.7),
-        }
+            system_prompt = kwargs.get("system_prompt", None)
+            user_prompt = kwargs.get("user_prompt", LLAMA_PARSER_PROMPT)
+
+        response = create_response(
+            api=api,
+            model=kwargs["model"],
+            system_prompt=system_prompt,
+            user_prompt=user_prompt,
+            image_url=image_url,
+            temperature=kwargs.get("temperature", 0.0),
+            max_tokens=kwargs.get("max_tokens", 1024),
+        )
 
         # Get completion from selected API
-        response = client.chat.completions.create(**completion_params)
-        token_usage = response.usage
+        page_text = response["response"]
+        token_usage = response["usage"]
 
-        # Extract the response text
-        page_text = response.choices[0].message.content
         if kwargs.get("verbose", None):
             logger.debug(f"Page {page_num + 1} response: {page_text}")
 
         # Extract content between output tags if present
         result = page_text
         if "<output>" in page_text:
-            result = page_text.split("<output>")[1].strip()
+            result = page_text.split("<output>")[-1].strip()
         if "</output>" in result:
             result = result.split("</output>")[0].strip()
         all_results.append(
             (
                 page_num,
                 result,
-                token_usage.prompt_tokens,
-                token_usage.completion_tokens,
-                token_usage.total_tokens,
+                token_usage["input_tokens"],
+                token_usage["output_tokens"],
+                token_usage["total_tokens"],
             )
         )
 
@@ -319,3 +449,28 @@ def parse_with_api(path: str, api: str, **kwargs) -> List[Dict] | str:
             "total": sum(total_tokens for _, _, _, _, total_tokens in all_results),
         },
     }
+
+
+def convert_doc_to_base64_images(path: str) -> List[Tuple[int, str]]:
+    """
+    Converts a document (PDF or image) to a base64 encoded string.
+
+    Args:
+        path (str): Path to the PDF file.
+
+    Returns:
+        str: Base64 encoded string of the PDF content.
+    """
+    if path.endswith(".pdf"):
+        pdf_document = pdfium.PdfDocument(path)
+        return [
+            (
+                page_num,
+                f"data:image/png;base64,{convert_pdf_page_to_base64(pdf_document, page_num)}",
+            )
+            for page_num in range(len(pdf_document))
+        ]
+    elif mimetypes.guess_type(path)[0].startswith("image"):
+        with open(path, "rb") as img_file:
+            image_base64 = base64.b64encode(img_file.read()).decode("utf-8")
+            return [(0, f"data:image/png;base64,{image_base64}")]

lexoid/core/parse_type/static_parser.py

@@ -1,12 +1,14 @@
 import os
 import re
 import tempfile
+from functools import wraps
 from time import time
 from typing import Dict, List
 
 import pandas as pd
 import pdfplumber
 from docx import Document
+from loguru import logger
 from pdfminer.high_level import extract_pages
 from pdfminer.layout import LTTextContainer
 from pdfplumber.utils import get_bbox_overlap, obj_to_bbox
@@ -22,6 +24,38 @@ from lexoid.core.utils import (
 )
 
 
+def retry_with_different_parser(func):
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except Exception as e:
+            if "pdfplumber" in kwargs.get("framework", "pdfplumber") and not kwargs.get(
+                "routed", False
+            ):
+                kwargs["framework"] = "pdfminer"
+                logger.warning(
+                    f"Retrying with pdfminer due to error: {e}. Original framework: {kwargs['framework']}"
+                )
+                return func(*args, **kwargs)
+            elif "pdfminer" in kwargs.get("framework", "pdfplumber") and not kwargs.get(
+                "routed", False
+            ):
+                kwargs["framework"] = "pdfplumber"
+                logger.warning(
+                    f"Retrying with pdfplumber due to error: {e}. Original framework: {kwargs['framework']}"
+                )
+                return func(*args, **kwargs)
+            else:
+                logger.error(
+                    f"Failed to parse document with both pdfplumber and pdfminer: {e}"
+                )
+                raise e
+
+    return wrapper
+
+
+@retry_with_different_parser
 def parse_static_doc(path: str, **kwargs) -> Dict:
     """
     Parses a document using static parsing methods.

lexoid/core/prompt_templates.py

@@ -41,7 +41,8 @@ Think step-by-step.
     '0' is typically more oval than 'O'
     '8' has a more angular top than 'B'
 {custom_instructions}
-- Return only the correct markdown without additional text or explanations. Do not any additional text (such as "```html" or "```markdown") in the output.
+- Return only the correct markdown without additional text or explanations.
+- DO NOT use code blocks such as "```html" or "```markdown" in the output unless there is a code block in the content.
 - Think before generating the output in <thinking></thinking> tags.
 
 Remember, your primary objective is to create an output that, when rendered, structurally replicates the original document's content as closely as possible without losing any textual details.

lexoid/core/utils.py

@@ -69,15 +69,45 @@ def convert_image_to_pdf(image_path: str) -> bytes:
 
 def remove_html_tags(text: str):
     html = markdown(text, extensions=["tables"])
-    return re.sub(HTML_TAG_PATTERN, "", html)
+    return re.sub(HTML_TAG_PATTERN, " ", html)
 
 
-def calculate_similarity(text1: str, text2: str, ignore_html=True) -> float:
+def clean_text(txt):
+    # Remove LaTeX commands (e.g. \command, \command[args]{args})
+    txt = re.sub(r"\\[a-zA-Z]+(\[[^\]]*\])?(\{[^}]*\})?", " ", txt)
+
+    # Replace all blocks of whitespace (including tabs and newlines) with a single space
+    txt = re.sub(r"\s+", " ", txt)
+
+    # Remove all non-alphanumeric characters except spaces
+    txt = re.sub(r"[^a-zA-Z0-9 ]", " ", txt)
+
+    return txt.strip()
+
+
+def calculate_similarity(
+    text1: str, text2: str, ignore_html: bool = True, diff_save_path: str = ""
+) -> float:
     """Calculate similarity ratio between two texts using SequenceMatcher."""
     if ignore_html:
         text1 = remove_html_tags(text1)
         text2 = remove_html_tags(text2)
-    return SequenceMatcher(None, text1, text2).ratio()
+
+    text1 = clean_text(clean_text(text1))
+    text2 = clean_text(clean_text(text2))
+
+    sm = SequenceMatcher(None, text1, text2)
+    # Save the diff and the texts for debugging
+    if diff_save_path:
+        with open(diff_save_path, "w") as f:
+            f.write(f"Text 1:\n{text1}\n\n")
+            f.write(f"Text 2:\n{text2}\n\n")
+            f.write("Differences:\n")
+            for tag, i1, i2, j1, j2 in sm.get_opcodes():
+                if tag == "equal":
+                    continue
+                f.write(f"{tag} {text1[i1:i2]} -> {text2[j1:j2]}\n")
+    return sm.ratio()
 
 
 def convert_pdf_page_to_image(
@@ -345,7 +375,7 @@ def get_webpage_soup(url: str) -> BeautifulSoup:
                 # Additional wait for any dynamic content
                 try:
                     await page.wait_for_selector("body", timeout=30000)
-                except:
+                except Exception:
                     pass
 
                 html = await page.content()
@@ -561,24 +591,32 @@ def router(path: str, priority: str = "speed") -> str:
         priority (str): The priority for routing: "accuracy" (preference to LLM_PARSE) or "speed" (preference to STATIC_PARSE).
     """
     file_type = get_file_type(path)
-    if file_type.startswith("text/") or "spreadsheet" in file_type or "presentation" in file_type:
+    if (
+        file_type.startswith("text/")
+        or "spreadsheet" in file_type
+        or "presentation" in file_type
+    ):
         return "STATIC_PARSE"
 
     if priority == "accuracy":
         # If the file is a PDF without images but has hyperlinks, use STATIC_PARSE
         # Otherwise, use LLM_PARSE
-        if (
-            file_type == "application/pdf"
-            and not has_image_in_pdf(path)
-            and has_hyperlink_in_pdf(path)
-        ):
+        has_image = has_image_in_pdf(path)
+        has_hyperlink = has_hyperlink_in_pdf(path)
+        if file_type == "application/pdf" and not has_image and has_hyperlink:
+            logger.debug("Using STATIC_PARSE for PDF with hyperlinks and no images.")
             return "STATIC_PARSE"
+        logger.debug(
+            f"Using LLM_PARSE because PDF has image ({has_image}) or has no hyperlink ({has_hyperlink})."
+        )
         return "LLM_PARSE"
     else:
         # If the file is a PDF without images, use STATIC_PARSE
         # Otherwise, use LLM_PARSE
         if file_type == "application/pdf" and not has_image_in_pdf(path):
+            logger.debug("Using STATIC_PARSE for PDF without images.")
             return "STATIC_PARSE"
+        logger.debug("Using LLM_PARSE because PDF has images")
         return "LLM_PARSE"
 
 

{lexoid-0.1.13.dist-info → lexoid-0.1.15.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lexoid
-Version: 0.1.13
+Version: 0.1.15
 Summary: 
 Requires-Python: >=3.10,<4.0
 Classifier: Programming Language :: Python :: 3
@@ -8,6 +8,7 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: anthropic (>=0.55.0,<0.56.0)
 Requires-Dist: bs4 (>=0.0.2,<0.0.3)
 Requires-Dist: docx2pdf (>=0.1.8,<0.2.0)
 Requires-Dist: google-generativeai (>=0.8.1,<0.9.0)
@@ -49,7 +50,8 @@ Description-Content-Type: text/markdown
 </div>
 
 [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/oidlabs-com/Lexoid/blob/main/examples/example_notebook_colab.ipynb)
-[![GitHub license](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://github.com/oidlabs-com/Lexoid/blob/main/LICENSE)
+[![Hugging Face](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-yellow)](https://huggingface.co/spaces/oidlabs/Lexoid)
+[![GitHub license](https://img.shields.io/badge/License-Apache_2.0-turquoise.svg)](https://github.com/oidlabs-com/Lexoid/blob/main/LICENSE)
 [![PyPI](https://img.shields.io/pypi/v/lexoid)](https://pypi.org/project/lexoid/)
 [![Docs](https://github.com/oidlabs-com/Lexoid/actions/workflows/deploy_docs.yml/badge.svg)](https://oidlabs-com.github.io/Lexoid/)
 
@@ -144,6 +146,7 @@ print(parsed_md)
 * Hugging Face
 * Together AI
 * OpenRouter
+* Fireworks
 
 ## Benchmark
 
@@ -151,22 +154,25 @@ Results aggregated across 5 iterations each for 5 documents.
 
 _Note:_ Benchmarks are currently done in the zero-shot setting.
 
-| Rank | Model                                                 | Mean Similarity | Std. Dev. | Time (s) | Cost($)  |
-| ---- | ----------------------------------------------------- | --------------- | --------- | -------- | -------- |
-| 1    | gemini-2.0-flash                                      | 0.829           | 0.102     | 7.41     | 0.000480 |
-| 2    | gemini-2.0-flash-001                                  | 0.814           | 0.176     | 6.85     | 0.000421 |
-| 3    | gemini-1.5-flash                                      | 0.797           | 0.143     | 9.54     | 0.000238 |
-| 4    | gemini-2.0-pro-exp                                    | 0.764           | 0.227     | 11.95    |   TBA    |
-| 5    | gemini-2.0-flash-thinking-exp                         | 0.746           | 0.266     | 10.46    |   TBA    |
-| 6    | gemini-1.5-pro                                        | 0.732           | 0.265     | 11.44    | 0.003332 |
-| 7    | gpt-4o                                                | 0.687           | 0.247     | 10.16    | 0.004736 |
-| 8    | gpt-4o-mini                                           | 0.642           | 0.213     | 9.71     | 0.000275 |
-| 9    | gemma-3-27b-it (via OpenRouter)                       | 0.628           | 0.299     | 18.79    | 0.000096 |
-| 10   | gemini-1.5-flash-8b                                   | 0.551           | 0.223     | 3.91     | 0.000055 |
-| 11   | Llama-Vision-Free (via Together AI)                   | 0.531           | 0.198     | 6.93     | 0        |
-| 12   | Llama-3.2-11B-Vision-Instruct-Turbo (via Together AI) | 0.524           | 0.192     | 3.68     | 0.000060 |
-| 13   | qwen/qwen-2.5-vl-7b-instruct (via OpenRouter)         | 0.482           | 0.209     | 11.53    | 0.000052 |
-| 14   | Llama-3.2-90B-Vision-Instruct-Turbo (via Together AI) | 0.461           | 0.306     | 19.26    | 0.000426 |
-| 15   | Llama-3.2-11B-Vision-Instruct (via Hugging Face)      | 0.451           | 0.257     | 4.54     |   0      |
-| 16   | microsoft/phi-4-multimodal-instruct (via OpenRouter)  | 0.366           | 0.287     | 10.80    | 0.000019 |
+| Rank | Model | Mean Similarity | Std. Dev. | Time (s) | Cost ($) |
+| --- | --- | --- | --- | --- | --- |
+| 1 | AUTO | 0.906 | 0.112 | 9.56 | 0.00068 |
+| 2 | gemini-2.0-flash | 0.897 | 0.126 | 9.91 | 0.00078 |
+| 3 | gemini-2.5-flash | 0.895 | 0.148 | 54.10 | 0.01051 |
+| 4 | gemini-1.5-pro | 0.868 | 0.283 | 15.03 | 0.00637 |
+| 5 | gemini-1.5-flash | 0.864 | 0.194 | 15.47 | 0.00044 |
+| 6 | claude-3-5-sonnet-20241022 | 0.851 | 0.209 | 15.99 | 0.01758 |
+| 7 | gemini-2.5-pro | 0.849 | 0.298 | 101.95 | 0.01859 |
+| 8 | claude-sonnet-4-20250514 | 0.804 | 0.190 | 19.27 | 0.02071 |
+| 9 | claude-opus-4-20250514 | 0.772 | 0.238 | 20.03 | 0.09207 |
+| 10 | accounts/fireworks/models/llama4-maverick-instruct-basic | 0.768 | 0.234 | 12.12 | 0.00150 |
+| 11 | gpt-4o | 0.748 | 0.284 | 26.80 | 0.01478 |
+| 12 | gpt-4o-mini | 0.733 | 0.231 | 18.18 | 0.00650 |
+| 13 | gpt-4.1-mini | 0.723 | 0.269 | 20.91 | 0.00351 |
+| 14 | google/gemma-3-27b-it | 0.681 | 0.334 | 19.41 | 0.00027 |
+| 15 | gpt-4.1 | 0.650 | 0.342 | 33.72 | 0.01443 |
+| 16 | claude-3-7-sonnet-20250219 | 0.633 | 0.369 | 14.24 | 0.01763 |
+| 17 | microsoft/phi-4-multimodal-instruct | 0.622 | 0.320 | 13.15 | 0.00050 |
+| 18 | qwen/qwen-2.5-vl-7b-instruct | 0.559 | 0.348 | 17.71 | 0.00086 |
+| 19 | meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo | 0.546 | 0.239 | 29.26 | 0.01103 |
 

lexoid-0.1.15.dist-info/RECORD

@@ -0,0 +1,9 @@
+lexoid/api.py,sha256=RxVJZ9JrQyJMKi-tx4GqdQoq_Bh956s90Fl8BVX15NY,15619
+lexoid/core/parse_type/llm_parser.py,sha256=KPB4JHXrvVASBIp9dGApM4j70P75tXffUsTP-SQTKZA,15506
+lexoid/core/parse_type/static_parser.py,sha256=5DMGDDYTTn5gXrqyz3JOxyLbVlhOj9Njk8n83pA0bFY,22534
+lexoid/core/prompt_templates.py,sha256=zftophGU0DNt0AKwYqN8jnAtH90-KlFTyJhs-I2yDiE,6351
+lexoid/core/utils.py,sha256=3goAgS8_8ybLxdhhqhDQDFyGeG_YytF45p81bVI42uI,21122
+lexoid-0.1.15.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+lexoid-0.1.15.dist-info/METADATA,sha256=cjNZsYCiCGVQ2GfuutCPsFNeO_iGONo5XtCMO4VU5DI,6235
+lexoid-0.1.15.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+lexoid-0.1.15.dist-info/RECORD,,

lexoid-0.1.13.dist-info/RECORD

@@ -1,9 +0,0 @@
-lexoid/api.py,sha256=lTkUcbGML29JrWJv4pE_ZqbzeJuHUE8b6OnijoLBEfU,11350
-lexoid/core/parse_type/llm_parser.py,sha256=rrc1Lwp-6ZAi8IVp3672mHAHUs1JefhT2rnYyQ1gA5E,11292
-lexoid/core/parse_type/static_parser.py,sha256=IovvF1GCLWFPh2-mwcgv6DpJmSVQBLnGcoIq7bwQ39Q,21299
-lexoid/core/prompt_templates.py,sha256=svSMH0yhm6ZjtOeTtUUEiCYi81ebVY9EZKPKP0Q921o,6311
-lexoid/core/utils.py,sha256=6s24X3-4Y57u70HzjIS798Tg8qx6Z3mLATf4xtENE-8,19718
-lexoid-0.1.13.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-lexoid-0.1.13.dist-info/METADATA,sha256=GHODqox4lX6qf_gjSy8ULYJZhaKKQ1BDKEUAOMi7R2U,6809
-lexoid-0.1.13.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-lexoid-0.1.13.dist-info/RECORD,,