doc-intelligence 0.0.7__tar.gz

doc_intelligence-0.0.7/.github/workflows/deploy-docs.yml

@@ -0,0 +1,40 @@
+name: Deploy MkDocs to GitHub Pages
+
+on:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+permissions:
+  contents: write
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+
+      - name: Install dependencies
+        run: uv sync
+
+      - name: Configure Git
+        run: |
+          git config user.name github-actions
+          git config user.email github-actions@github.com
+
+      - name: Deploy to GitHub Pages
+        run: uv run mkdocs gh-deploy --force

doc_intelligence-0.0.7/.github/workflows/release.yml

@@ -0,0 +1,29 @@
+name: Publish to PyPI
+
+on:
+  push:
+    tags:
+      - 'v*'
+
+jobs:
+  release:
+    name: Publish to PyPI
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Install uv and setup the python version
+        uses: astral-sh/setup-uv@v7
+
+      - name: Install Python 3.10
+        run: uv python install 3.10
+
+      - name: Build
+        run: uv build
+
+      - name: Publish package
+        run: uv publish

doc_intelligence-0.0.7/.gitignore

@@ -0,0 +1,13 @@
+# Python-generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+
+# Virtual environments
+.venv
+.env
+
+data/

doc_intelligence-0.0.7/.pre-commit-config.yaml

@@ -0,0 +1,17 @@
+repos:
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  # Ruff version.
+  rev: v0.14.13
+  hooks:
+    # Run the linter.
+    - id: ruff-check
+      args: [ --fix ]
+    # Run the formatter.
+    - id: ruff-format
+# - repo: https://github.com/facebook/pyrefly-pre-commit
+#   rev: 0.48.1  # Note: this is the version of the pre-commit hook, NOT the pyrefly version used for type checking
+#   hooks:
+#     - id: pyrefly-check
+#       name: Pyrefly (type checking)
+#       pass_filenames: false  # Recommended to do full repo checks. However, you can change this to `true` to only check changed files
+#       language: system  # Use system-installed pyrefly

doc_intelligence-0.0.7/.python-version

@@ -0,0 +1 @@
+3.10

doc_intelligence-0.0.7/.vscode/settings.json

@@ -0,0 +1,15 @@
+{
+    "[python]": {
+      "editor.formatOnSave": true,
+      "editor.codeActionsOnSave": {
+        "source.fixAll": "explicit",
+        "source.organizeImports": "explicit"
+      },
+      "editor.defaultFormatter": "charliermarsh.ruff"
+    },
+    "ruff.enable": true,
+    "ruff.organizeImports": true,
+    "ruff.fixAll": true
+  }
+  
+  

doc_intelligence-0.0.7/PKG-INFO

@@ -0,0 +1,151 @@
+Metadata-Version: 2.4
+Name: doc-intelligence
+Version: 0.0.7
+Summary: Document AI - Intelligent document processing and extraction
+Requires-Python: >=3.10
+Requires-Dist: google-genai>=1.57.0
+Requires-Dist: langchain>=1.2.9
+Requires-Dist: loguru>=0.7.3
+Requires-Dist: mkdocs-material>=9.7.1
+Requires-Dist: mkdocs>=1.6.1
+Requires-Dist: openai>=2.15.0
+Requires-Dist: pdfplumber>=0.11.9
+Requires-Dist: pre-commit>=4.5.1
+Requires-Dist: pymupdf>=1.26.7
+Requires-Dist: pytest>=9.0.2
+Requires-Dist: python-dotenv>=1.2.1
+Description-Content-Type: text/markdown
+
+# Document AI
+
+**Documentation:** [https://zeel-04.github.io/document-ai/](https://zeel-04.github.io/document-ai/)
+
+A library for parsing, formatting, and processing documents that can be used to build AI-powered document processing pipelines with structured data extraction and citation support.
+
+## Features
+
+- Extract structured data from PDF documents using LLMs
+- Automatic citation tracking with page numbers, line numbers, and bounding boxes
+- Support for digital PDFs
+- Type-safe data models using Pydantic
+- OpenAI integration with support for reasoning models
+
+## Installation
+
+### Requirements
+
+- Python >= 3.10
+- OpenAI API key
+
+### Install uv
+
+First, install [uv](https://docs.astral.sh/uv/) if you haven't already:
+
+```bash
+curl -LsSf https://astral.sh/uv/install.sh | sh
+```
+
+### Install from Source
+
+Clone the repository and install the package:
+
+```bash
+git clone https://github.com/zeel-04/document-ai.git
+cd document-ai
+uv sync
+```
+
+### Install from Git (Alternative)
+
+You can also install directly from the git repository:
+
+```bash
+uv pip install git+https://github.com/zeel-04/document-ai.git
+```
+
+## Quick Start
+
+Set up your OpenAI API key:
+
+```bash
+echo "OPENAI_API_KEY=your-api-key-here" > .env
+```
+
+Here's a simple example to extract structured data from a PDF:
+
+```python
+from dotenv import load_dotenv
+from document_ai.processer import DocumentProcessor
+from document_ai.llm import OpenAILLM
+from pydantic import BaseModel
+
+# Load environment variables
+load_dotenv()
+
+# Initialize the LLM
+llm = OpenAILLM()
+
+# Create a processor from a PDF file
+processor = DocumentProcessor.from_digital_pdf(
+    uri="path/to/your/document.pdf",
+    llm=llm,
+)
+
+# Define your data model
+class Balance(BaseModel):
+    ending_balance: float
+
+# Configure extraction with citations
+config = {
+    "response_format": Balance,
+    "llm_config": {
+        "model": "gpt-5",
+        "reasoning": {"effort": "minimal"},
+    },
+    "extraction_config": {
+        "include_citations": True,
+        "extraction_mode": "single_pass",
+        "page_numbers": [0, 1],  # Optional: specify which pages to process
+    }
+}
+
+# Extract structured data
+response = processor.extract(config)
+
+# Get the extracted data and citations
+data, citations = response
+print(f"Extracted data: {data}")
+print(f"Citations: {citations}")
+```
+
+### Sample Output
+
+The `extract` method returns a tuple containing the extracted data and citation information:
+
+```python
+(Balance(ending_balance=111.61),
+ {'ending_balance': {'value': 111.61,
+   'citations': [{'page': 0,
+     'bboxes': [{'x0': 0.058823529411764705,
+       'top': 0.6095707475757575,
+       'x1': 0.5635455037254902,
+       'bottom': 0.6221969596969696}]}]}})
+```
+
+## Documentation
+
+For more detailed documentation, see the [docs](./docs) directory or visit the [documentation site](https://zeel-04.github.io/document-ai/).
+
+## Development Setup
+
+Prerequisites:
+
+- Python 3.10+
+- uv
+
+```bash
+git clone https://github.com/zeel-04/document-ai.git
+cd document-ai
+uv venv
+uv sync 
+```

doc_intelligence-0.0.7/README.md

@@ -0,0 +1,133 @@
+# Document AI
+
+**Documentation:** [https://zeel-04.github.io/document-ai/](https://zeel-04.github.io/document-ai/)
+
+A library for parsing, formatting, and processing documents that can be used to build AI-powered document processing pipelines with structured data extraction and citation support.
+
+## Features
+
+- Extract structured data from PDF documents using LLMs
+- Automatic citation tracking with page numbers, line numbers, and bounding boxes
+- Support for digital PDFs
+- Type-safe data models using Pydantic
+- OpenAI integration with support for reasoning models
+
+## Installation
+
+### Requirements
+
+- Python >= 3.10
+- OpenAI API key
+
+### Install uv
+
+First, install [uv](https://docs.astral.sh/uv/) if you haven't already:
+
+```bash
+curl -LsSf https://astral.sh/uv/install.sh | sh
+```
+
+### Install from Source
+
+Clone the repository and install the package:
+
+```bash
+git clone https://github.com/zeel-04/document-ai.git
+cd document-ai
+uv sync
+```
+
+### Install from Git (Alternative)
+
+You can also install directly from the git repository:
+
+```bash
+uv pip install git+https://github.com/zeel-04/document-ai.git
+```
+
+## Quick Start
+
+Set up your OpenAI API key:
+
+```bash
+echo "OPENAI_API_KEY=your-api-key-here" > .env
+```
+
+Here's a simple example to extract structured data from a PDF:
+
+```python
+from dotenv import load_dotenv
+from document_ai.processer import DocumentProcessor
+from document_ai.llm import OpenAILLM
+from pydantic import BaseModel
+
+# Load environment variables
+load_dotenv()
+
+# Initialize the LLM
+llm = OpenAILLM()
+
+# Create a processor from a PDF file
+processor = DocumentProcessor.from_digital_pdf(
+    uri="path/to/your/document.pdf",
+    llm=llm,
+)
+
+# Define your data model
+class Balance(BaseModel):
+    ending_balance: float
+
+# Configure extraction with citations
+config = {
+    "response_format": Balance,
+    "llm_config": {
+        "model": "gpt-5",
+        "reasoning": {"effort": "minimal"},
+    },
+    "extraction_config": {
+        "include_citations": True,
+        "extraction_mode": "single_pass",
+        "page_numbers": [0, 1],  # Optional: specify which pages to process
+    }
+}
+
+# Extract structured data
+response = processor.extract(config)
+
+# Get the extracted data and citations
+data, citations = response
+print(f"Extracted data: {data}")
+print(f"Citations: {citations}")
+```
+
+### Sample Output
+
+The `extract` method returns a tuple containing the extracted data and citation information:
+
+```python
+(Balance(ending_balance=111.61),
+ {'ending_balance': {'value': 111.61,
+   'citations': [{'page': 0,
+     'bboxes': [{'x0': 0.058823529411764705,
+       'top': 0.6095707475757575,
+       'x1': 0.5635455037254902,
+       'bottom': 0.6221969596969696}]}]}})
+```
+
+## Documentation
+
+For more detailed documentation, see the [docs](./docs) directory or visit the [documentation site](https://zeel-04.github.io/document-ai/).
+
+## Development Setup
+
+Prerequisites:
+
+- Python 3.10+
+- uv
+
+```bash
+git clone https://github.com/zeel-04/document-ai.git
+cd document-ai
+uv venv
+uv sync 
+```

doc_intelligence-0.0.7/docs/index.md

@@ -0,0 +1,5 @@
+# Document AI
+
+A library for parsing, formatting, and processing documents that can be used to build AI-powered document processing pipelines.
+
+> Please refer to the [quickstart](quickstart.md) for a guide on how to get started.

doc_intelligence-0.0.7/docs/quickstart.md

@@ -0,0 +1,125 @@
+# Quickstart
+
+This guide will help you get started with Document AI.
+
+## Installation
+
+## Requirements
+
+- Python >= 3.10
+- OpenAI API key
+
+### Install uv
+
+First, install [uv](https://docs.astral.sh/uv/) if you haven't already:
+
+```bash
+curl -LsSf https://astral.sh/uv/install.sh | sh
+```
+
+### Install from Source
+
+Clone the repository and install the package:
+
+```bash
+# Clone the repository
+git clone https://github.com/zeel-04/document-ai.git
+cd document-ai
+
+# Install the package with uv
+uv sync
+```
+
+### Install from Git (Alternative)
+
+You can also install directly from the git repository:
+
+```bash
+uv pip install git+https://github.com/zeel-04/document-ai.git
+```
+
+## Environment Setup
+
+Document AI uses OpenAI's API for document processing. Set up your API key:
+
+```bash
+# Create a .env file
+echo "OPENAI_API_KEY=your-api-key-here" > .env
+```
+
+## Basic Usage
+
+Here's a simple example to extract structured data from a PDF document:
+
+```python
+from dotenv import load_dotenv
+from document_ai.processer import DocumentProcessor
+from document_ai.llm import OpenAILLM
+from pydantic import BaseModel
+
+# Load environment variables
+load_dotenv()
+
+# Initialize the LLM
+llm = OpenAILLM()
+
+# Create a processor from a PDF file
+processor = DocumentProcessor.from_digital_pdf(
+    uri="path/to/your/document.pdf",
+    llm=llm,
+)
+
+# Define your data model
+class Balance(BaseModel):
+    ending_balance: float
+
+# Configure extraction with citations
+config = {
+    "response_format": Balance,
+    "llm_config": {
+        "model": "gpt-5",
+        "reasoning": {"effort": "minimal"},
+    },
+    "extraction_config": {
+        "include_citations": True,
+        "extraction_mode": "single_pass",
+        "page_numbers": [0, 1],  # Optional: specify which pages to process
+    }
+}
+
+# Extract structured data
+response = processor.extract(config)
+
+# Get the extracted data and citations
+data, citations = response
+print(f"Extracted data: {data}")
+print(f"Citations: {citations}")
+```
+
+### Sample Output
+
+The `extract` method returns a tuple containing:
+1. The extracted data as a Pydantic model instance
+2. A dictionary with citation information for each field
+
+```python
+# Example output
+(Balance(ending_balance=111.61),
+ {'ending_balance': {'value': 111.61,
+   'citations': [{'page': 0,
+     'bboxes': [{'x0': 0.058823529411764705,
+       'top': 0.6095707475757575,
+       'x1': 0.5635455037254902,
+       'bottom': 0.6221969596969696}]}]}})
+```
+
+### Configuration Options
+
+- **response_format**: Your Pydantic model class
+- **llm_config**: 
+  - `model`: The OpenAI model to use (e.g., "gpt-5", "gpt-4o")
+  - `reasoning`: Optional reasoning configuration with `effort` level ("minimal", "low", "medium", "high")
+- **extraction_config**:
+  - `include_citations`: Set to `True` to get citation information
+  - `extraction_mode`: "single_pass" for single-pass extraction
+  - `page_numbers`: Optional list of page indices to process (0-indexed)

doc_intelligence-0.0.7/document_ai/__init__.py

@@ -0,0 +1 @@
+from document_ai.base import BaseFormatter, BaseParser, Document

doc_intelligence-0.0.7/document_ai/base.py

@@ -0,0 +1,64 @@
+from abc import ABC, abstractmethod
+from typing import Any
+
+from langchain_core.output_parsers import JsonOutputParser
+
+from .schemas.core import Document, PydanticModel
+
+
+class BaseParser(ABC):
+    @abstractmethod
+    def parse(self, document: Document) -> PydanticModel:
+        pass
+
+
+class BaseFormatter(ABC):
+    @abstractmethod
+    def format_document_for_llm(
+        self,
+        document: Document,
+        **kwargs,
+    ) -> str:
+        pass
+
+
+class BaseLLM(ABC):
+    @abstractmethod
+    def generate_structured_output(
+        self,
+        model: str,
+        messages: list[dict[str, str]],
+        reasoning: Any,
+        output_format: type[PydanticModel],
+        openai_text: dict[str, Any] | None = None,
+    ) -> PydanticModel | None:
+        pass
+
+    @abstractmethod
+    def generate_text(
+        self,
+        system_prompt: str,
+        user_prompt: str,
+        **kwargs,
+    ) -> str:
+        pass
+
+
+class BaseExtractor(ABC):
+    def __init__(
+        self,
+        llm: BaseLLM,
+    ):
+        self.llm = llm
+        self.json_parser = JsonOutputParser()
+
+    @abstractmethod
+    def extract(
+        self,
+        document: Document,
+        llm_config: dict[str, Any],
+        extraction_config: dict[str, Any],
+        formatter: BaseFormatter,
+        response_format: type[PydanticModel],
+    ) -> tuple[PydanticModel, dict[str, Any] | None]:
+        pass

doc_intelligence-0.0.7/document_ai/config.py

@@ -0,0 +1,7 @@
+config = {
+    "digital_pdf": {
+        "llm": {
+            "model": "gpt-5-mini",
+        },
+    },
+}

doc_intelligence-0.0.7/document_ai/extractor.py

@@ -0,0 +1,76 @@
+from typing import Any
+
+from loguru import logger
+
+from .base import BaseExtractor, BaseFormatter
+from .llm import BaseLLM
+from .pydantic_to_json_instance_schema import (
+    pydantic_to_json_instance_schema,
+    stringify_schema,
+)
+from .schemas.core import Document, PydanticModel
+from .types.pdf import PDFExtractionMode
+from .utils import enrich_citations_with_bboxes, strip_citations
+
+
+class DigitalPDFExtractor(BaseExtractor):
+    def __init__(self, llm: BaseLLM):
+        super().__init__(llm)
+
+        self.system_prompt = """Act as an expert in the field of document extraction and information extraction from documents."""
+        self.user_prompt = """Your job is to extract structured mentioned in schema data from a document given below.
+
+DOCUMENT:
+{content_text}
+
+OUTPUT SCHEMA:
+{schema}
+
+Generate output in JSON format.
+"""
+
+    def extract(
+        self,
+        document: Document,
+        llm_config: dict[str, Any],
+        extraction_config: dict[str, Any],
+        formatter: BaseFormatter,
+        response_format: type[PydanticModel],
+    ) -> tuple[PydanticModel, dict[str, Any] | None]:
+        if document.extraction_mode == PDFExtractionMode.SINGLE_PASS:
+            json_instance_schema = stringify_schema(
+                pydantic_to_json_instance_schema(
+                    response_format,
+                    citation=document.include_citations,
+                    citation_level="line",
+                )
+            )
+            logger.debug(
+                f"DigitalPDFExtractor: extract: json_instance_schema: {json_instance_schema}"
+            )
+            content_text = formatter.format_document_for_llm(
+                document, **extraction_config
+            )
+            logger.debug(f"DigitalPDFExtractor: extract: content_text: {content_text}")
+            user_prompt = self.user_prompt.format(
+                content_text=content_text, schema=json_instance_schema
+            )
+
+            response = self.llm.generate_text(
+                system_prompt=self.system_prompt,
+                user_prompt=user_prompt,
+                **llm_config,
+            )
+
+        if document.extraction_mode == PDFExtractionMode.MULTI_PASS:
+            raise NotImplementedError("Multi-pass extraction is not implemented yet")
+
+        response_dict = self.json_parser.parse(response)
+
+        if document.include_citations:
+            response_metadata = enrich_citations_with_bboxes(response_dict, document)
+            response_dict = strip_citations(response_metadata)
+        else:
+            response_metadata = None
+
+        return response_format(**response_dict), response_metadata