lexoid 0.1.12__py3-none-any.whl → 0.1.14__py3-none-any.whl

lexoid/api.py

@@ -10,7 +10,11 @@ from typing import Union, Dict, List
 
 from loguru import logger
 
-from lexoid.core.parse_type.llm_parser import parse_llm_doc
+from lexoid.core.parse_type.llm_parser import (
+    parse_llm_doc,
+    create_response,
+    convert_doc_to_base64_images,
+)
 from lexoid.core.parse_type.static_parser import parse_static_doc
 from lexoid.core.utils import (
     convert_to_pdf,
@@ -49,6 +53,7 @@ def parse_chunk(path: str, parser_type: ParserType, **kwargs) -> Dict:
             - parent_title: Title of parent doc if recursively parsed
             - recursive_docs: List of dictionaries for recursively parsed documents
             - token_usage: Dictionary containing token usage statistics
+            - parser_used: Which parser was actually used
     """
     if parser_type == ParserType.AUTO:
         router_priority = kwargs.get("router_priority", "speed")
@@ -60,10 +65,13 @@ def parse_chunk(path: str, parser_type: ParserType, **kwargs) -> Dict:
     )
     if parser_type == ParserType.STATIC_PARSE:
         logger.debug("Using static parser")
-        return parse_static_doc(path, **kwargs)
+        result = parse_static_doc(path, **kwargs)
     else:
         logger.debug("Using LLM parser")
-        return parse_llm_doc(path, **kwargs)
+        result = parse_llm_doc(path, **kwargs)
+
+    result["parser_used"] = parser_type
+    return result
 
 
 def parse_chunk_list(
@@ -82,15 +90,18 @@ def parse_chunk_list(
     """
     combined_segments = []
     raw_texts = []
-    token_usage = {"input": 0, "output": 0, "image_count": 0}
+    token_usage = {"input": 0, "output": 0, "llm_page_count": 0}
     for file_path in file_paths:
         result = parse_chunk(file_path, parser_type, **kwargs)
         combined_segments.extend(result["segments"])
         raw_texts.append(result["raw"])
-        if "token_usage" in result:
+        if (
+            result.get("parser_used") == ParserType.LLM_PARSE
+            and "token_usage" in result
+        ):
             token_usage["input"] += result["token_usage"]["input"]
             token_usage["output"] += result["token_usage"]["output"]
-            token_usage["image_count"] += len(result["segments"])
+            token_usage["llm_page_count"] += len(result["segments"])
     token_usage["total"] = token_usage["input"] + token_usage["output"]
 
     return {
@@ -136,7 +147,7 @@ def parse(
     as_pdf = kwargs.get("as_pdf", False)
     depth = kwargs.get("depth", 1)
 
-    if type(parser_type) == str:
+    if type(parser_type) is str:
         parser_type = ParserType[parser_type]
     if (
         path.lower().endswith((".doc", ".docx"))
@@ -184,7 +195,7 @@ def parse(
 
         if not path.lower().endswith(".pdf") or parser_type == ParserType.STATIC_PARSE:
             kwargs["split"] = False
-            result = parse_chunk(path, parser_type, **kwargs)
+            result = parse_chunk_list([path], parser_type, kwargs)
         else:
             kwargs["split"] = True
             split_dir = os.path.join(temp_dir, "splits/")
@@ -219,42 +230,43 @@ def parse(
                 "token_usage": {
                     "input": sum(r["token_usage"]["input"] for r in chunk_results),
                     "output": sum(r["token_usage"]["output"] for r in chunk_results),
-                    "image_count": sum(
-                        r["token_usage"]["image_count"] for r in chunk_results
+                    "llm_page_count": sum(
+                        r["token_usage"]["llm_page_count"] for r in chunk_results
                     ),
                     "total": sum(r["token_usage"]["total"] for r in chunk_results),
                 },
             }
 
-            if "api_cost_mapping" in kwargs:
-                api_cost_mapping = kwargs["api_cost_mapping"]
-                if isinstance(api_cost_mapping, dict):
-                    api_cost_mapping = api_cost_mapping
-                elif isinstance(api_cost_mapping, str) and os.path.exists(
-                    api_cost_mapping
-                ):
-                    with open(api_cost_mapping, "r") as f:
-                        api_cost_mapping = json.load(f)
-                else:
-                    raise ValueError(f"Unsupported API cost value: {api_cost_mapping}.")
-
-                api_cost = api_cost_mapping.get(
-                    kwargs.get("model", "gemini-2.0-flash"), None
+        if "api_cost_mapping" in kwargs and "token_usage" in result:
+            api_cost_mapping = kwargs["api_cost_mapping"]
+            if isinstance(api_cost_mapping, dict):
+                api_cost_mapping = api_cost_mapping
+            elif isinstance(api_cost_mapping, str) and os.path.exists(api_cost_mapping):
+                with open(api_cost_mapping, "r") as f:
+                    api_cost_mapping = json.load(f)
+            else:
+                raise ValueError(f"Unsupported API cost value: {api_cost_mapping}.")
+
+            api_cost = api_cost_mapping.get(
+                kwargs.get("model", "gemini-2.0-flash"), None
+            )
+            if api_cost:
+                token_usage = result["token_usage"]
+                token_cost = {
+                    "input": token_usage["input"] * api_cost["input"] / 1_000_000,
+                    "input-image": api_cost.get("input-image", 0)
+                    * token_usage.get("llm_page_count", 0),
+                    "output": token_usage["output"] * api_cost["output"] / 1_000_000,
+                }
+                token_cost["total"] = (
+                    token_cost["input"]
+                    + token_cost["input-image"]
+                    + token_cost["output"]
                 )
-                if api_cost:
-                    token_usage = result["token_usage"]
-                    token_cost = {
-                        "input": token_usage["input"] * api_cost["input"] / 1_000_000
-                        + api_cost.get("input-image", 0) * token_usage["image_count"],
-                        "output": token_usage["output"]
-                        * api_cost["output"]
-                        / 1_000_000,
-                    }
-                    token_cost["total"] = token_cost["input"] + token_cost["output"]
-                    result["token_cost"] = token_cost
-
-            if as_pdf:
-                result["pdf_path"] = path
+                result["token_cost"] = token_cost
+
+        if as_pdf:
+            result["pdf_path"] = path
 
     if depth > 1:
         recursive_docs = []
@@ -285,3 +297,63 @@ def parse(
         result["recursive_docs"] = recursive_docs
 
     return result
+
+
+def parse_with_schema(
+    path: str, schema: Dict, api: str = "openai", model: str = "gpt-4o-mini", **kwargs
+) -> List[List[Dict]]:
+    """
+    Parses a PDF using an LLM to generate structured output conforming to a given JSON schema.
+
+    Args:
+        path (str): Path to the PDF file.
+        schema (Dict): JSON schema to which the parsed output should conform.
+        api (str, optional): LLM API provider (One of "openai", "huggingface", "together", "openrouter", and "fireworks").
+        model (str, optional): LLM model name.
+        **kwargs: Additional arguments for the parser (e.g.: temperature, max_tokens).
+
+    Returns:
+        List[List[Dict]]: List of dictionaries for each page, each conforming to the provided schema.
+    """
+    system_prompt = f"""
+        The output should be formatted as a JSON instance that conforms to the JSON schema below.
+
+        As an example, for the schema {{
+        "properties": {{
+            "foo": {{
+            "title": "Foo",
+            "description": "a list of strings",
+            "type": "array",
+            "items": {{"type": "string"}}
+            }}
+        }},
+        "required": ["foo"]
+        }}, the object {{"foo": ["bar", "baz"]}} is valid. The object {{"properties": {{"foo": ["bar", "baz"]}}}} is not.
+
+        Here is the output schema:
+        {json.dumps(schema, indent=2)}
+
+        """
+
+    user_prompt = "You are an AI agent that parses documents and returns them in the specified JSON format. Please parse the document and return it in the required format."
+
+    responses = []
+    images = convert_doc_to_base64_images(path)
+    for i, (page_num, image) in enumerate(images):
+        resp_dict = create_response(
+            api=api,
+            model=model,
+            user_prompt=user_prompt,
+            system_prompt=system_prompt,
+            image_url=image,
+            temperature=kwargs.get("temperature", 0.0),
+            max_tokens=kwargs.get("max_tokens", 1024),
+        )
+
+        response = resp_dict.get("response", "")
+        response = response.split("```json")[-1].split("```")[0].strip()
+        logger.debug(f"Processing page {page_num + 1} with response: {response}")
+        new_dict = json.loads(response)
+        responses.append(new_dict)
+
+    return responses

lexoid/core/parse_type/llm_parser.py

@@ -3,23 +3,24 @@ import io
 import mimetypes
 import os
 import time
+from functools import wraps
+from typing import Dict, List, Optional, Tuple
+
 import pypdfium2 as pdfium
 import requests
-from functools import wraps
+from huggingface_hub import InferenceClient
+from loguru import logger
+from openai import OpenAI
 from requests.exceptions import HTTPError
-from typing import Dict, List
+from together import Together
 
 from lexoid.core.prompt_templates import (
     INSTRUCTIONS_ADD_PG_BREAK,
+    LLAMA_PARSER_PROMPT,
     OPENAI_USER_PROMPT,
     PARSER_PROMPT,
-    LLAMA_PARSER_PROMPT,
 )
 from lexoid.core.utils import convert_image_to_pdf
-from loguru import logger
-from openai import OpenAI
-from together import Together
-from huggingface_hub import InferenceClient
 
 
 def retry_on_http_error(func):
@@ -65,10 +66,13 @@ def parse_llm_doc(path: str, **kwargs) -> List[Dict] | str:
         return parse_with_api(path, api="huggingface", **kwargs)
     if any(model.startswith(prefix) for prefix in ["microsoft", "google", "qwen"]):
         return parse_with_api(path, api="openrouter", **kwargs)
+    if model.startswith("accounts/fireworks"):
+        return parse_with_api(path, api="fireworks", **kwargs)
     raise ValueError(f"Unsupported model: {model}")
 
 
 def parse_with_gemini(path: str, **kwargs) -> List[Dict] | str:
+    logger.debug(f"Parsing with Gemini API and model {kwargs['model']}")
     api_key = os.environ.get("GOOGLE_API_KEY")
     if not api_key:
         raise ValueError("GOOGLE_API_KEY environment variable is not set")
@@ -105,7 +109,7 @@ def parse_with_gemini(path: str, **kwargs) -> List[Dict] | str:
             }
         ],
         "generationConfig": {
-            "temperature": kwargs.get("temperature", 0.7),
+            "temperature": kwargs.get("temperature", 0.2),
         },
     }
 
@@ -127,7 +131,7 @@ def parse_with_gemini(path: str, **kwargs) -> List[Dict] | str:
 
     combined_text = ""
     if "<output>" in raw_text:
-        combined_text = raw_text.split("<output>")[1].strip()
+        combined_text = raw_text.split("<output>")[-1].strip()
     if "</output>" in result:
         combined_text = result.split("</output>")[0].strip()
 
@@ -169,18 +173,54 @@ def convert_pdf_page_to_base64(
     return base64.b64encode(img_byte_arr.getvalue()).decode("utf-8")
 
 
-def parse_with_api(path: str, api: str, **kwargs) -> List[Dict] | str:
-    """
-    Parse documents (PDFs or images) using various vision model APIs.
+def get_messages(
+    system_prompt: Optional[str], user_prompt: Optional[str], image_url: Optional[str]
+) -> List[Dict]:
+    messages = []
+    if system_prompt:
+        messages.append(
+            {
+                "role": "system",
+                "content": system_prompt,
+            }
+        )
+    base_message = (
+        [
+            {"type": "text", "text": user_prompt},
+        ]
+        if user_prompt
+        else []
+    )
+    image_message = (
+        [
+            {
+                "type": "image_url",
+                "image_url": {"url": image_url},
+            }
+        ]
+        if image_url
+        else []
+    )
 
-    Args:
-        path (str): Path to the document to parse
-        api (str): Which API to use ("openai", "huggingface", or "together")
-        **kwargs: Additional arguments including model, temperature, title, etc.
+    messages.append(
+        {
+            "role": "user",
+            "content": base_message + image_message,
+        }
+    )
 
-    Returns:
-        Dict: Dictionary containing parsed document data
-    """
+    return messages
+
+
+def create_response(
+    api: str,
+    model: str,
+    system_prompt: Optional[str] = None,
+    user_prompt: Optional[str] = None,
+    image_url: Optional[str] = None,
+    temperature: float = 0.2,
+    max_tokens: int = 1024,
+) -> Dict:
     # Initialize appropriate client
     clients = {
         "openai": lambda: OpenAI(),
@@ -192,11 +232,52 @@ def parse_with_api(path: str, api: str, **kwargs) -> List[Dict] | str:
             base_url="https://openrouter.ai/api/v1",
             api_key=os.environ["OPENROUTER_API_KEY"],
         ),
+        "fireworks": lambda: OpenAI(
+            base_url="https://api.fireworks.ai/inference/v1",
+            api_key=os.environ["FIREWORKS_API_KEY"],
+        ),
     }
     assert api in clients, f"Unsupported API: {api}"
-    logger.debug(f"Parsing with {api} API and model {kwargs['model']}")
     client = clients[api]()
 
+    # Prepare messages for the API call
+    messages = get_messages(system_prompt, user_prompt, image_url)
+
+    # Common completion parameters
+    completion_params = {
+        "model": model,
+        "messages": messages,
+        "max_tokens": max_tokens,
+        "temperature": temperature,
+    }
+
+    # Get completion from selected API
+    response = client.chat.completions.create(**completion_params)
+    token_usage = response.usage
+
+    # Extract the response text
+    page_text = response.choices[0].message.content
+
+    return {
+        "response": page_text,
+        "usage": token_usage,
+    }
+
+
+def parse_with_api(path: str, api: str, **kwargs) -> List[Dict] | str:
+    """
+    Parse documents (PDFs or images) using various vision model APIs.
+
+    Args:
+        path (str): Path to the document to parse
+        api (str): Which API to use ("openai", "huggingface", or "together")
+        **kwargs: Additional arguments including model, temperature, title, etc.
+
+    Returns:
+        Dict: Dictionary containing parsed document data
+    """
+    logger.debug(f"Parsing with {api} API and model {kwargs['model']}")
+
     # Handle different input types
     mime_type, _ = mimetypes.guess_type(path)
     if mime_type and mime_type.startswith("image"):
@@ -215,67 +296,39 @@ def parse_with_api(path: str, api: str, **kwargs) -> List[Dict] | str:
             for page_num in range(len(pdf_document))
         ]
 
-    # API-specific message formatting
-    def get_messages(page_num: int, image_url: str) -> List[Dict]:
-        image_message = {
-            "type": "image_url",
-            "image_url": {"url": image_url},
-        }
-
+    # Process each page/image
+    all_results = []
+    for page_num, image_url in images:
         if api == "openai":
             system_prompt = kwargs.get(
                 "system_prompt", PARSER_PROMPT.format(custom_instructions="")
             )
             user_prompt = kwargs.get("user_prompt", OPENAI_USER_PROMPT)
-            return [
-                {
-                    "role": "system",
-                    "content": system_prompt,
-                },
-                {
-                    "role": "user",
-                    "content": [
-                        {"type": "text", "text": user_prompt},
-                        image_message,
-                    ],
-                },
-            ]
         else:
-            prompt = kwargs.get("system_prompt", LLAMA_PARSER_PROMPT)
-            base_message = {"type": "text", "text": prompt}
-            return [
-                {
-                    "role": "user",
-                    "content": [base_message, image_message],
-                }
-            ]
-
-    # Process each page/image
-    all_results = []
-    for page_num, image_url in images:
-        messages = get_messages(page_num, image_url)
-
-        # Common completion parameters
-        completion_params = {
-            "model": kwargs["model"],
-            "messages": messages,
-            "max_tokens": kwargs.get("max_tokens", 1024),
-            "temperature": kwargs.get("temperature", 0.7),
-        }
+            system_prompt = kwargs.get("system_prompt", None)
+            user_prompt = kwargs.get("user_prompt", LLAMA_PARSER_PROMPT)
+
+        response = create_response(
+            api=api,
+            model=kwargs["model"],
+            system_prompt=system_prompt,
+            user_prompt=user_prompt,
+            image_url=image_url,
+            temperature=kwargs.get("temperature", 0.2),
+            max_tokens=kwargs.get("max_tokens", 1024),
+        )
 
         # Get completion from selected API
-        response = client.chat.completions.create(**completion_params)
-        token_usage = response.usage
+        page_text = response["response"]
+        token_usage = response["usage"]
 
-        # Extract the response text
-        page_text = response.choices[0].message.content
         if kwargs.get("verbose", None):
             logger.debug(f"Page {page_num + 1} response: {page_text}")
 
         # Extract content between output tags if present
         result = page_text
         if "<output>" in page_text:
-            result = page_text.split("<output>")[1].strip()
+            result = page_text.split("<output>")[-1].strip()
         if "</output>" in result:
             result = result.split("</output>")[0].strip()
         all_results.append(
@@ -319,3 +372,28 @@ def parse_with_api(path: str, api: str, **kwargs) -> List[Dict] | str:
             "total": sum(total_tokens for _, _, _, _, total_tokens in all_results),
         },
     }
+
+
+def convert_doc_to_base64_images(path: str) -> List[Tuple[int, str]]:
+    """
+    Converts a document (PDF or image) to a base64 encoded string.
+
+    Args:
+        path (str): Path to the PDF file.
+
+    Returns:
+        str: Base64 encoded string of the PDF content.
+    """
+    if path.endswith(".pdf"):
+        pdf_document = pdfium.PdfDocument(path)
+        return [
+            (
+                page_num,
+                f"data:image/png;base64,{convert_pdf_page_to_base64(pdf_document, page_num)}",
+            )
+            for page_num in range(len(pdf_document))
+        ]
+    elif mimetypes.guess_type(path)[0].startswith("image"):
+        with open(path, "rb") as img_file:
+            image_base64 = base64.b64encode(img_file.read()).decode("utf-8")
+            return [(0, f"data:image/png;base64,{image_base64}")]

lexoid/core/parse_type/static_parser.py

@@ -1,7 +1,8 @@
 import os
+import re
 import tempfile
 from time import time
-from typing import List, Dict
+from typing import Dict, List
 
 import pandas as pd
 import pdfplumber
@@ -9,14 +10,15 @@ from docx import Document
 from pdfminer.high_level import extract_pages
 from pdfminer.layout import LTTextContainer
 from pdfplumber.utils import get_bbox_overlap, obj_to_bbox
-from pptx2md import convert, ConversionConfig
+from pptx2md import ConversionConfig, convert
+
 
 from lexoid.core.utils import (
     get_file_type,
     get_uri_rect,
     html_to_markdown,
-    split_pdf,
     split_md_by_headings,
+    split_pdf,
 )
 
 
@@ -203,6 +205,25 @@ def embed_links_in_text(page, text, links):
     return text
 
 
+def detect_indentation_level(word, base_left_position):
+    """Determine indentation level based on left position difference."""
+    left_diff = word["x0"] - base_left_position
+    if left_diff < 5:
+        return 0
+    return int(left_diff // 25) + 1
+
+
+def embed_email_links(text: str) -> str:
+    """
+    Detect email addresses in text and wrap them in angle brackets.
+    For example, 'mail@example.com' becomes '<mail@example.com>'.
+    """
+    email_pattern = re.compile(
+        r"(?<![<\[])(?P<email>\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b)(?![>\]])"
+    )
+    return email_pattern.sub(lambda match: f"<{match.group('email')}>", text)
+
+
 def process_pdf_page_with_pdfplumber(page, uri_rects, **kwargs):
     """
     Process a single page's content and return formatted markdown text.
@@ -213,7 +234,26 @@ def process_pdf_page_with_pdfplumber(page, uri_rects, **kwargs):
     last_y = None
     x_tolerance = kwargs.get("x_tolerance", 1)
     y_tolerance = kwargs.get("y_tolerance", 5)
-
+    next_h_line_idx = 0
+
+    # First detect horizontal lines that could be markdown rules
+    horizontal_lines = []
+    if hasattr(page, "lines"):
+        for line in page.lines:
+            # Check if line is approximately horizontal (within 5 degrees)
+            if (
+                abs(line["height"]) < 0.1
+                or abs(line["width"]) > abs(line["height"]) * 20
+            ):
+                # Consider it a horizontal rule candidate
+                horizontal_lines.append(
+                    {
+                        "top": line["top"],
+                        "bottom": line["bottom"],
+                        "x0": line["x0"],
+                        "x1": line["x1"],
+                    }
+                )
     # Table settings
     vertical_strategy = kwargs.get("vertical_strategy", "lines")
     horizontal_strategy = kwargs.get("horizontal_strategy", "lines")
@@ -243,14 +283,43 @@ def process_pdf_page_with_pdfplumber(page, uri_rects, **kwargs):
         extra_attrs=["size", "top", "bottom", "fontname"],
     )
 
-    def format_paragraph(text_elements):
-        """Format a paragraph with styling applied to individual words"""
-        formatted_words = []
-        for element in text_elements:
-            text = element["text"]
-            formatting = get_text_formatting(element)
-            formatted_words.append(apply_markdown_formatting(text, formatting))
-        return f"{' '.join(formatted_words)}\n\n"
+    if words:
+        font_sizes = [w.get("size", 12) for w in words]
+        body_font_size = max(set(font_sizes), key=font_sizes.count)
+    else:
+        body_font_size = 12
+
+    left_positions = []
+    prev_bottom = None
+
+    for word in words:
+        # Check if this is likely a new line (first word in line)
+        if prev_bottom is None or abs(word["top"] - prev_bottom) > y_tolerance:
+            left_positions.append(word["x0"])
+        prev_bottom = word["top"]
+
+    # Find the most common minimum left position (mode)
+    if left_positions:
+        base_left = max(set(left_positions), key=left_positions.count)
+    else:
+        base_left = 0
+
+    for line in horizontal_lines:
+        # Check each word to see if it overlaps with this line
+        for word in words:
+            # Get word bounding box coordinates
+            word_left = word["x0"]
+            word_right = word["x1"]
+            word_top = word["top"]
+            word_bottom = word["bottom"]
+
+            # Check if word overlaps with line in both x and y dimensions
+            x_overlap = (word_left <= line["x1"]) and (word_right >= line["x0"])
+            y_overlap = (word_top <= line["bottom"]) and (word_bottom >= line["top"])
+
+            if x_overlap and y_overlap:
+                word["text"] = f"~~{word['text']}~~"
+                break
 
     def get_text_formatting(word):
         """
@@ -260,19 +329,22 @@ def process_pdf_page_with_pdfplumber(page, uri_rects, **kwargs):
         formatting = {
             "bold": False,
             "italic": False,
+            "monospace": False,
         }
-
         # Check font name for common bold/italic indicators
         font_name = word.get("fontname", "").lower()
         if any(style in font_name for style in ["bold", "heavy", "black"]):
             formatting["bold"] = True
         if any(style in font_name for style in ["italic", "oblique"]):
             formatting["italic"] = True
-
+        if "mono" in font_name:  # Detect monospace fonts
+            formatting["monospace"] = True
         return formatting
 
     def apply_markdown_formatting(text, formatting):
         """Apply markdown formatting to text based on detected styles"""
+        if formatting["monospace"]:
+            text = f"`{text}`"
         if formatting["bold"] and formatting["italic"]:
             text = f"***{text}***"
         elif formatting["bold"]:
@@ -281,12 +353,64 @@ def process_pdf_page_with_pdfplumber(page, uri_rects, **kwargs):
             text = f"*{text}*"
         return text
 
-    def detect_heading_level(font_size):
-        if font_size >= 24:
+    def format_paragraph(text_elements):
+        """
+        Format a paragraph with styling applied to individual words.
+        If all words are monospace, treat the paragraph as a code block.
+        Otherwise, wrap monospace words with backticks (`).
+        """
+
+        all_monospace = True
+        formatted_words = []
+
+        for element in text_elements:
+            if isinstance(element, tuple) and element[0] == "indent":
+                indent = "&nbsp;" * element[1] * 3
+                formatted_words.append(indent)
+                continue
+
+            text = element["text"]
+            formatting = get_text_formatting(element)
+
+            if formatting.get("monospace", False):
+                # Wrap monospace words with backticks
+                formatted_words.append(f"`{text}`")
+            else:
+                all_monospace = False
+                # Apply other markdown formatting
+                formatted_words.append(apply_markdown_formatting(text, formatting))
+
+        # If all words are monospace, format as a code block
+        if all_monospace:
+            if isinstance(text_elements[0], tuple):
+                indent_str = " " * text_elements[0][1]
+                if len(text_elements) > 1:
+                    text_elements = text_elements[1:]
+                    text_elements[0]["text"] = indent_str + text_elements[0]["text"]
+                else:
+                    return indent_str
+            code_content = " ".join([element["text"] for element in text_elements])
+            return f"```\n{code_content}\n```\n\n"
+
+        # Otherwise, return the formatted paragraph
+        return f"{' '.join(formatted_words)}\n\n"
+
+    def detect_heading_level(font_size, body_font_size):
+        """Determine heading level based on font size ratio.
+
+        Args:
+            font_size: The font size to evaluate
+            body_font_size: The base body font size for comparison
+
+        Returns:
+            int: The heading level (1-3) or None if not a heading
+        """
+        size_ratio = font_size / body_font_size
+        if size_ratio >= 2:
             return 1
-        elif font_size >= 20:
+        elif size_ratio >= 1.4:
             return 2
-        elif font_size >= 16:
+        elif size_ratio >= 1.2:
             return 3
         return None
 
@@ -303,18 +427,41 @@ def process_pdf_page_with_pdfplumber(page, uri_rects, **kwargs):
             )
         )
     tables.sort(key=lambda x: x[1]["bottom"])
+
     content_elements = []
-    for word in words:
+    for line in horizontal_lines:
+        content_elements.append(
+            (
+                "horizontal_line",
+                {
+                    "top": line["top"],
+                    "bottom": line["bottom"],
+                    "x0": line["x0"],
+                    "x1": line["x1"],
+                },
+            )
+        )
+
+    for i, word in enumerate(words):
         while tables and word["bottom"] > tables[0][1]["bottom"]:
             content_elements.append(tables.pop(0))
+
+        # Equate position of words on the same line
+        if i > 0 and abs(word["top"] - words[i - 1]["top"]) < 3:
+            word["top"] = words[i - 1]["top"]
+
         content_elements.append(("word", word))
     content_elements.extend(tables)
 
+    content_elements.sort(
+        key=lambda x: x[1]["top"] if isinstance(x[1], dict) and "top" in x[1] else 0
+    )
+
     for element_type, element in content_elements:
+        # If there are any pending paragraphs or headings, add them first
         if element_type == "table":
-            # If there are any pending paragraphs or headings, add them first
             if current_heading:
-                level = detect_heading_level(current_heading[0]["size"])
+                level = detect_heading_level(current_heading[0]["size"], body_font_size)
                 heading_text = format_paragraph(current_heading)
                 markdown_content.append(f"{'#' * level} {heading_text}")
                 current_heading = []
@@ -324,11 +471,22 @@ def process_pdf_page_with_pdfplumber(page, uri_rects, **kwargs):
             # Add the table
             markdown_content.append(element["content"])
             last_y = element["bottom"]
+        elif element_type == "horizontal_line":
+            while (next_h_line_idx < len(horizontal_lines)) and (
+                last_y is not None
+                and horizontal_lines[next_h_line_idx]["top"] <= last_y
+            ):
+                # Insert the horizontal rule *after* the preceding text
+                if current_paragraph:  # Flush any pending paragraph
+                    markdown_content.append(format_paragraph(current_paragraph))
+                    current_paragraph = []
+                markdown_content.append("\n---\n\n")  # Add the rule
+                next_h_line_idx += 1
         else:
             # Process word
             word = element
             # Check if this might be a heading
-            heading_level = detect_heading_level(word["size"])
+            heading_level = detect_heading_level(word["size"], body_font_size)
 
             # Detect new line based on vertical position
             is_new_line = last_y is not None and abs(word["top"] - last_y) > y_tolerance
@@ -336,7 +494,9 @@ def process_pdf_page_with_pdfplumber(page, uri_rects, **kwargs):
             if is_new_line:
                 # If we were collecting a heading
                 if current_heading:
-                    level = detect_heading_level(current_heading[0]["size"])
+                    level = detect_heading_level(
+                        current_heading[0]["size"], body_font_size
+                    )
                     heading_text = format_paragraph(current_heading)
                     markdown_content.append(f"{'#' * level} {heading_text}")
                     current_heading = []
@@ -346,6 +506,9 @@ def process_pdf_page_with_pdfplumber(page, uri_rects, **kwargs):
                     markdown_content.append(format_paragraph(current_paragraph))
                     current_paragraph = []
 
+                indent_level = detect_indentation_level(word, base_left)
+                current_paragraph.append(("indent", indent_level))
+
             # Add word to appropriate collection
             if heading_level:
                 if current_paragraph:  # Flush any pending paragraph
@@ -354,7 +517,9 @@ def process_pdf_page_with_pdfplumber(page, uri_rects, **kwargs):
                 current_heading.append(word)
             else:
                 if current_heading:  # Flush any pending heading
-                    level = detect_heading_level(current_heading[0]["size"])
+                    level = detect_heading_level(
+                        current_heading[0]["size"], body_font_size
+                    )
                     heading_text = format_paragraph(current_heading)
                     markdown_content.append(f"{'#' * level} {heading_text}")
                     current_heading = []
@@ -364,7 +529,7 @@ def process_pdf_page_with_pdfplumber(page, uri_rects, **kwargs):
 
     # Handle remaining content
     if current_heading:
-        level = detect_heading_level(current_heading[0]["size"])
+        level = detect_heading_level(current_heading[0]["size"], body_font_size)
         heading_text = format_paragraph(current_heading)
         markdown_content.append(f"{'#' * level} {heading_text}")
 
@@ -383,8 +548,15 @@ def process_pdf_page_with_pdfplumber(page, uri_rects, **kwargs):
         if links:
             content = embed_links_in_text(page, content, links)
 
+    content = embed_email_links(content)
+
     # Remove redundant formatting
-    content = content.replace("** **", " ").replace("* *", " ")
+    content = (
+        content.replace("** **", " ")
+        .replace("* *", " ")
+        .replace("` `", " ")
+        .replace("\n```\n\n```", "")
+    )
 
     return content
 

lexoid/core/prompt_templates.py

@@ -41,7 +41,8 @@ Think step-by-step.
     '0' is typically more oval than 'O'
     '8' has a more angular top than 'B'
 {custom_instructions}
-- Return only the correct markdown without additional text or explanations. Do not any additional text (such as "```html" or "```markdown") in the output.
+- Return only the correct markdown without additional text or explanations.
+- DO NOT use code blocks such as "```html" or "```markdown" in the output unless there is a code block in the content.
 - Think before generating the output in <thinking></thinking> tags.
 
 Remember, your primary objective is to create an output that, when rendered, structurally replicates the original document's content as closely as possible without losing any textual details.

lexoid/core/utils.py

@@ -345,7 +345,7 @@ def get_webpage_soup(url: str) -> BeautifulSoup:
                 # Additional wait for any dynamic content
                 try:
                     await page.wait_for_selector("body", timeout=30000)
-                except:
+                except Exception:
                     pass
 
                 html = await page.content()
@@ -561,24 +561,32 @@ def router(path: str, priority: str = "speed") -> str:
         priority (str): The priority for routing: "accuracy" (preference to LLM_PARSE) or "speed" (preference to STATIC_PARSE).
     """
     file_type = get_file_type(path)
-    if file_type.startswith("text/") or "spreadsheet" in file_type or "presentation" in file_type:
+    if (
+        file_type.startswith("text/")
+        or "spreadsheet" in file_type
+        or "presentation" in file_type
+    ):
         return "STATIC_PARSE"
 
     if priority == "accuracy":
         # If the file is a PDF without images but has hyperlinks, use STATIC_PARSE
         # Otherwise, use LLM_PARSE
-        if (
-            file_type == "application/pdf"
-            and not has_image_in_pdf(path)
-            and has_hyperlink_in_pdf(path)
-        ):
+        has_image = has_image_in_pdf(path)
+        has_hyperlink = has_hyperlink_in_pdf(path)
+        if file_type == "application/pdf" and not has_image and has_hyperlink:
+            logger.debug("Using STATIC_PARSE for PDF with hyperlinks and no images.")
             return "STATIC_PARSE"
+        logger.debug(
+            f"Using LLM_PARSE because PDF has image ({has_image}) or has no hyperlink ({has_hyperlink})."
+        )
         return "LLM_PARSE"
     else:
         # If the file is a PDF without images, use STATIC_PARSE
         # Otherwise, use LLM_PARSE
         if file_type == "application/pdf" and not has_image_in_pdf(path):
+            logger.debug("Using STATIC_PARSE for PDF without images.")
             return "STATIC_PARSE"
+        logger.debug("Using LLM_PARSE because PDF has images")
         return "LLM_PARSE"
 
 

{lexoid-0.1.12.dist-info → lexoid-0.1.14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lexoid
-Version: 0.1.12
+Version: 0.1.14
 Summary: 
 Requires-Python: >=3.10,<4.0
 Classifier: Programming Language :: Python :: 3
@@ -49,7 +49,8 @@ Description-Content-Type: text/markdown
 </div>
 
 [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/oidlabs-com/Lexoid/blob/main/examples/example_notebook_colab.ipynb)
-[![GitHub license](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://github.com/oidlabs-com/Lexoid/blob/main/LICENSE)
+[![Hugging Face](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-yellow)](https://huggingface.co/spaces/oidlabs/Lexoid)
+[![GitHub license](https://img.shields.io/badge/License-Apache_2.0-turquoise.svg)](https://github.com/oidlabs-com/Lexoid/blob/main/LICENSE)
 [![PyPI](https://img.shields.io/pypi/v/lexoid)](https://pypi.org/project/lexoid/)
 [![Docs](https://github.com/oidlabs-com/Lexoid/actions/workflows/deploy_docs.yml/badge.svg)](https://oidlabs-com.github.io/Lexoid/)
 
@@ -144,6 +145,7 @@ print(parsed_md)
 * Hugging Face
 * Together AI
 * OpenRouter
+* Fireworks
 
 ## Benchmark
 
@@ -151,22 +153,25 @@ Results aggregated across 5 iterations each for 5 documents.
 
 _Note:_ Benchmarks are currently done in the zero-shot setting.
 
-| Rank | Model                                                 | Mean Similarity | Std. Dev. | Time (s) | Cost($)  |
-| ---- | ----------------------------------------------------- | --------------- | --------- | -------- | -------- |
-| 1    | gemini-2.0-flash                                      | 0.829           | 0.102     | 7.41     | 0.000480 |
-| 2    | gemini-2.0-flash-001                                  | 0.814           | 0.176     | 6.85     | 0.000421 |
-| 3    | gemini-1.5-flash                                      | 0.797           | 0.143     | 9.54     | 0.000238 |
-| 4    | gemini-2.0-pro-exp                                    | 0.764           | 0.227     | 11.95    |   TBA    |
-| 5    | gemini-2.0-flash-thinking-exp                         | 0.746           | 0.266     | 10.46    |   TBA    |
-| 6    | gemini-1.5-pro                                        | 0.732           | 0.265     | 11.44    | 0.003332 |
-| 7    | gpt-4o                                                | 0.687           | 0.247     | 10.16    | 0.004736 |
-| 8    | gpt-4o-mini                                           | 0.642           | 0.213     | 9.71     | 0.000275 |
-| 9    | gemma-3-27b-it (via OpenRouter)                       | 0.628           | 0.299     | 18.79    | 0.000096 |
-| 10   | gemini-1.5-flash-8b                                   | 0.551           | 0.223     | 3.91     | 0.000055 |
-| 11   | Llama-Vision-Free (via Together AI)                   | 0.531           | 0.198     | 6.93     | 0        |
-| 12   | Llama-3.2-11B-Vision-Instruct-Turbo (via Together AI) | 0.524           | 0.192     | 3.68     | 0.000060 |
-| 13   | qwen/qwen-2.5-vl-7b-instruct (via OpenRouter)         | 0.482           | 0.209     | 11.53    | 0.000052 |
-| 14   | Llama-3.2-90B-Vision-Instruct-Turbo (via Together AI) | 0.461           | 0.306     | 19.26    | 0.000426 |
-| 15   | Llama-3.2-11B-Vision-Instruct (via Hugging Face)      | 0.451           | 0.257     | 4.54     |   0      |
-| 16   | microsoft/phi-4-multimodal-instruct (via OpenRouter)  | 0.366           | 0.287     | 10.80    | 0.000019 |
+| Rank | Model | Mean Similarity | Std. Dev. | Time (s) | Cost ($) |
+| --- | --- | --- | --- | --- | --- |
+| 1 | gemini-2.0-flash | 0.829 | 0.102 | 7.41 | 0.00048 |
+| 2 | gemini-2.0-flash-001 | 0.814 | 0.176 | 6.85 | 0.000421 |
+| 3 | gemini-1.5-flash | 0.797 | 0.143 | 9.54 | 0.000238 |
+| 4 | gemini-2.0-pro-exp | 0.764 | 0.227 | 11.95 | TBA |
+| 5 | AUTO | 0.76 | 0.184 | 5.14 | 0.000217 |
+| 6 | gemini-2.0-flash-thinking-exp | 0.746 | 0.266 | 10.46 | TBA |
+| 7 | gemini-1.5-pro | 0.732 | 0.265 | 11.44 | 0.003332 |
+| 8 | accounts/fireworks/models/llama4-maverick-instruct-basic (via Fireworks) | 0.687 | 0.221 | 8.07 | 0.000419 |
+| 9 | gpt-4o | 0.687 | 0.247 | 10.16 | 0.004736 |
+| 10 | accounts/fireworks/models/llama4-scout-instruct-basic (via Fireworks) | 0.675 | 0.184 | 5.98 | 0.000226 |
+| 11 | gpt-4o-mini | 0.642 | 0.213 | 9.71 | 0.000275 |
+| 12 | gemma-3-27b-it (via OpenRouter) | 0.628 | 0.299 | 18.79 | 0.000096 |
+| 13 | gemini-1.5-flash-8b | 0.551 | 0.223 | 3.91 | 0.000055 |
+| 14 | Llama-Vision-Free (via Together AI) | 0.531 | 0.198 | 6.93 | 0 |
+| 15 | Llama-3.2-11B-Vision-Instruct-Turbo (via Together AI) | 0.524 | 0.192 | 3.68 | 0.00006 |
+| 16 | qwen/qwen-2.5-vl-7b-instruct (via OpenRouter) | 0.482 | 0.209 | 11.53 | 0.000052 |
+| 17 | Llama-3.2-90B-Vision-Instruct-Turbo (via Together AI) | 0.461 | 0.306 | 19.26 | 0.000426 |
+| 18 | Llama-3.2-11B-Vision-Instruct (via Hugging Face) | 0.451 | 0.257 | 4.54 | 0 |
+| 19 | microsoft/phi-4-multimodal-instruct (via OpenRouter) | 0.366 | 0.287 | 10.8 | 0.000019 |
 

lexoid-0.1.14.dist-info/RECORD

@@ -0,0 +1,9 @@
+lexoid/api.py,sha256=le24OVKpPZ4UYs3X77HsFjfRTTu50jdcLLDcp_Vn5-M,13924
+lexoid/core/parse_type/llm_parser.py,sha256=gkjIuBa1IJ59FejLqMcwuvpzlm2VIotWQk02INQbiA4,13234
+lexoid/core/parse_type/static_parser.py,sha256=IovvF1GCLWFPh2-mwcgv6DpJmSVQBLnGcoIq7bwQ39Q,21299
+lexoid/core/prompt_templates.py,sha256=zftophGU0DNt0AKwYqN8jnAtH90-KlFTyJhs-I2yDiE,6351
+lexoid/core/utils.py,sha256=R0zyOzprtDIwwR4XNhE0qzi1FBuLZxFsR4uKUCrtWtI,20138
+lexoid-0.1.14.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+lexoid-0.1.14.dist-info/METADATA,sha256=o5543YUA6EF6iN7dTNWneQfzPeIeadpHTSqtnq3IV7Q,6383
+lexoid-0.1.14.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+lexoid-0.1.14.dist-info/RECORD,,

lexoid-0.1.12.dist-info/RECORD

@@ -1,9 +0,0 @@
-lexoid/api.py,sha256=lTkUcbGML29JrWJv4pE_ZqbzeJuHUE8b6OnijoLBEfU,11350
-lexoid/core/parse_type/llm_parser.py,sha256=rrc1Lwp-6ZAi8IVp3672mHAHUs1JefhT2rnYyQ1gA5E,11292
-lexoid/core/parse_type/static_parser.py,sha256=v4GWUmZVBBIF9TnbkhPBt2gspk0Oq_ujtNGnXZHLBr8,15055
-lexoid/core/prompt_templates.py,sha256=svSMH0yhm6ZjtOeTtUUEiCYi81ebVY9EZKPKP0Q921o,6311
-lexoid/core/utils.py,sha256=6s24X3-4Y57u70HzjIS798Tg8qx6Z3mLATf4xtENE-8,19718
-lexoid-0.1.12.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-lexoid-0.1.12.dist-info/METADATA,sha256=XMHFMqwDj2DgSaZcZjXU881NxdPsRGBAsUyPyRsJvyU,6809
-lexoid-0.1.12.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-lexoid-0.1.12.dist-info/RECORD,,