chunkr-ai 0.0.1__tar.gz → 0.0.2__tar.gz

chunkr_ai-0.0.2/PKG-INFO

@@ -0,0 +1,16 @@
+Metadata-Version: 2.2
+Name: chunkr-ai
+Version: 0.0.2
+Summary: Python client for chunkr: open source document intelligence
+Author-email: Ishaan Kapoor <ishaan@lumina.sh>
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: build>=1.2.2.post1
+Requires-Dist: httpx>=0.28.1
+Requires-Dist: pillow>=11.1.0
+Requires-Dist: pydantic>=2.10.4
+Requires-Dist: python-dotenv>=1.0.1
+Requires-Dist: requests>=2.32.3
+Requires-Dist: twine>=6.0.1
+Provides-Extra: test
+Requires-Dist: pytest>=8.3.4; extra == "test"

chunkr_ai-0.0.2/pyproject.toml

@@ -0,0 +1,25 @@
+[build-system]
+requires = ["setuptools>=42", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "chunkr-ai"
+version = "0.0.2"
+authors = [{"name" = "Ishaan Kapoor", "email" = "ishaan@lumina.sh"}]
+description = "Python client for chunkr: open source document intelligence"
+readme = "README.md"
+license = {"file" = "LICENSE"}
+dependencies = [
+    "build>=1.2.2.post1",
+    "httpx>=0.28.1",
+    "pillow>=11.1.0",
+    "pydantic>=2.10.4",
+    "python-dotenv>=1.0.1",
+    "requests>=2.32.3",
+    "twine>=6.0.1",
+]
+
+[project.optional-dependencies]
+test = [
+    "pytest>=8.3.4",
+]

chunkr_ai-0.0.2/src/chunkr_ai/__init__.py

@@ -0,0 +1,4 @@
+from .api.chunkr import Chunkr
+from .api.chunkr_async import ChunkrAsync
+
+__all__ = ['Chunkr', 'ChunkrAsync']

chunkr_ai-0.0.2/src/chunkr_ai/api/auth.py

@@ -0,0 +1,14 @@
+from typing import Optional
+
+class HeadersMixin:
+    """Mixin class for handling authorization headers"""
+    
+    def get_api_key(self) -> str:
+        """Get the API key"""
+        if not hasattr(self, '_api_key') or not self._api_key:
+            raise ValueError("API key not set")
+        return self._api_key
+        
+    def _headers(self) -> dict:
+        """Generate authorization headers"""
+        return {"Authorization": self.get_api_key()} 

chunkr_ai-0.0.2/src/chunkr_ai/api/chunkr.py

@@ -0,0 +1,125 @@
+from .models import TaskResponse, Configuration
+from .auth import HeadersMixin
+from dotenv import load_dotenv
+import io
+import os
+from pathlib import Path
+from PIL import Image
+import requests
+from typing import Union, BinaryIO, Tuple
+
+class Chunkr(HeadersMixin):
+    """Client for interacting with the Chunkr API."""
+
+    def __init__(self, url: str = None, api_key: str = None):
+        load_dotenv()
+        self.url = (
+            url or 
+            os.getenv('CHUNKR_URL') or 
+            'https://api.chunkr.ai' 
+        )
+        self._api_key = (
+            api_key or 
+            os.getenv('CHUNKR_API_KEY')
+        )
+        if not self._api_key:
+            raise ValueError("API key must be provided either directly, in .env file, or as CHUNKR_API_KEY environment variable. You can get an api key at: https://www.chunkr.ai")
+            
+        self.url = self.url.rstrip("/")
+
+    def _prepare_file(
+        self,
+        file: Union[str, BinaryIO, Image.Image, bytes, io.BytesIO]
+    ) -> Tuple[str, BinaryIO]:
+        """Convert various file types into a tuple of (filename, file-like object).
+
+        Args:
+            file: Input file in various formats
+
+        Returns:
+            Tuple[str, BinaryIO]: Filename and file-like object ready for upload
+        """
+        if isinstance(file, str):
+            path = Path(file).resolve() 
+            if not path.exists():
+                raise FileNotFoundError(f"File not found: {file}")
+            return path.name, path.open("rb")
+        elif isinstance(file, Image.Image):
+            img_byte_arr = io.BytesIO()
+            file.save(img_byte_arr, format=file.format or 'PNG')
+            img_byte_arr.seek(0)
+            return "image.png", img_byte_arr
+        elif isinstance(file, bytes):
+            return "document", io.BytesIO(file)
+        elif isinstance(file, io.BytesIO):
+            return "document", file
+        else:
+            return "document", file
+
+    def upload(self, file: Union[str, BinaryIO, Image.Image, bytes, io.BytesIO], config: Configuration = None) -> TaskResponse:
+        """Upload a file and wait for processing to complete.
+
+        The file can be one of:
+        - str: Path to a file on disk
+        - BinaryIO: A file-like object (e.g., opened with 'rb' mode)
+        - Image.Image: A PIL/Pillow Image object
+        - bytes: Raw binary data
+        - io.BytesIO: A binary stream in memory
+
+        Args:
+            file: The file to upload.
+            config:
+                Configuration options for processing. Optional.
+
+        Returns:
+            TaskResponse: The completed task response
+        """
+        return self.start_upload(file, config).poll()
+
+    def start_upload(self, file: Union[str, BinaryIO, Image.Image, bytes, io.BytesIO], config: Configuration = None) -> TaskResponse:
+        """Upload a file for processing and immediately return the task response. It will not wait for processing to complete. To wait for the full processing to complete, use `task.poll()`
+
+        The file can be one of:
+        - str: Path to a file on disk
+        - BinaryIO: A file-like object (e.g., opened with 'rb' mode)
+        - Image.Image: A PIL/Pillow Image object
+        - bytes: Raw binary data
+        - io.BytesIO: A binary stream in memory
+
+        Args:
+            file: The file to upload.
+            config (Configuration, optional): Configuration options for processing
+
+        Returns:
+            TaskResponse: The initial task response
+
+        Raises:
+            requests.exceptions.HTTPError: If the API request fails
+        """
+        url = f"{self.url}/api/v1/task"
+        filename, file_obj = self._prepare_file(file)
+        
+        files = {"file": (filename, file_obj)}   
+        r = requests.post(
+            url, 
+            files=files, 
+            json=config.dict() if config else {}, 
+            headers=self._headers()
+        )
+        r.raise_for_status()
+        return TaskResponse(**r.json()).with_api_key(self._api_key)
+
+    def get_task(self, task_id: str) -> TaskResponse:
+        """Get a task response by its ID.
+        
+        Args:
+            task_id (str): The ID of the task to get
+
+        Returns:
+            TaskResponse: The task response
+        """
+        url = f"{self.url}/api/v1/task/{task_id}"
+        r = requests.get(url, headers=self._headers())
+        r.raise_for_status()
+        return TaskResponse(**r.json()).with_api_key(self._api_key)
+

chunkr_ai-0.0.2/src/chunkr_ai/api/chunkr_async.py

@@ -0,0 +1,39 @@
+from .chunkr import Chunkr
+from .models import TaskResponse, Configuration
+import httpx
+import io
+from PIL import Image
+from typing import Union, BinaryIO
+
+class ChunkrAsync(Chunkr):
+    """Async client for interacting with the Chunkr API.
+    
+    This class inherits from the Chunkr class but works with async HTTP requests.
+    """
+
+    async def upload(self, file: Union[str, BinaryIO, Image.Image, bytes, io.BytesIO], config: Configuration = None) -> TaskResponse:
+        task = await self.start_upload(file, config)
+        return await task.poll_async()
+
+    async def start_upload(self, file: Union[str, BinaryIO, Image.Image, bytes, io.BytesIO], config: Configuration = None) -> TaskResponse:
+        url = f"{self.url}/api/v1/task"
+        filename, file_obj = self._prepare_file(file)
+        async with httpx.AsyncClient() as client:
+            files = {"file": (filename, file_obj)}
+            r = await client.post(
+                url, 
+                files=files, 
+                json=config.dict() if config else {},
+                headers=self._headers()
+            )
+            r.raise_for_status()
+            return TaskResponse(**r.json()).with_api_key(self._api_key)
+
+    async def get_task(self, task_id: str) -> TaskResponse:
+        url = f"{self.url}/api/v1/task/{task_id}"
+        async with httpx.AsyncClient() as client:
+            r = await client.get(url, headers=self._headers())
+            r.raise_for_status()
+            return TaskResponse(**r.json()).with_api_key(self._api_key)
+
+ 

chunkr_ai-0.0.2/src/chunkr_ai/api/models.py

@@ -0,0 +1,231 @@
+from .auth import HeadersMixin
+import asyncio
+from datetime import datetime
+from enum import Enum
+import httpx
+from pydantic import BaseModel, Field, PrivateAttr
+import requests
+import time
+from typing import Optional, List, Dict, Union
+
+class GenerationStrategy(str, Enum):
+    LLM = "LLM"
+    AUTO = "Auto"
+
+class CroppingStrategy(str, Enum):
+    ALL = "All" 
+    AUTO = "Auto"
+
+class LlmConfig(BaseModel):
+    model: str
+    prompt: str
+    temperature: float = 0.0
+
+class AutoGenerationConfig(BaseModel):
+    html: GenerationStrategy = GenerationStrategy.AUTO
+    llm: Optional[LlmConfig] = None
+    markdown: GenerationStrategy = GenerationStrategy.AUTO
+    crop_image: CroppingStrategy = CroppingStrategy.ALL
+
+class LlmGenerationConfig(BaseModel):
+    html: GenerationStrategy = GenerationStrategy.LLM
+    llm: Optional[LlmConfig] = None
+    markdown: GenerationStrategy = GenerationStrategy.LLM
+    crop_image: CroppingStrategy = CroppingStrategy.ALL
+
+class SegmentProcessing(BaseModel):
+    title: AutoGenerationConfig = Field(default_factory=AutoGenerationConfig)
+    section_header: AutoGenerationConfig = Field(default_factory=AutoGenerationConfig)
+    text: AutoGenerationConfig = Field(default_factory=AutoGenerationConfig)
+    list_item: AutoGenerationConfig = Field(default_factory=AutoGenerationConfig)
+    table: LlmGenerationConfig = Field(default_factory=LlmGenerationConfig)
+    picture: AutoGenerationConfig = Field(default_factory=AutoGenerationConfig)
+    caption: AutoGenerationConfig = Field(default_factory=AutoGenerationConfig)
+    formula: LlmGenerationConfig = Field(default_factory=LlmGenerationConfig)
+    footnote: AutoGenerationConfig = Field(default_factory=AutoGenerationConfig)
+    page_header: AutoGenerationConfig = Field(default_factory=AutoGenerationConfig)
+    page_footer: AutoGenerationConfig = Field(default_factory=AutoGenerationConfig)
+    page: AutoGenerationConfig = Field(default_factory=AutoGenerationConfig)
+
+class ChunkProcessing(BaseModel):
+    target_length: int = 512
+
+class Property(BaseModel):
+    name: str
+    title: Optional[str]
+    prop_type: str
+    description: Optional[str]
+    default: Optional[str]
+
+class JsonSchema(BaseModel):
+    title: str
+    properties: List[Property]
+    schema_type: Optional[str]
+
+class OcrStrategy(str, Enum):
+    ALL = "All"
+    AUTO = "Auto"
+
+class SegmentationStrategy(str, Enum):
+    LAYOUT_ANALYSIS = "LayoutAnalysis"
+    PAGE = "Page"
+
+class BoundingBox(BaseModel):
+    left: float
+    top: float
+    width: float
+    height: float
+
+class OCRResult(BaseModel):
+    bbox: BoundingBox
+    text: str
+    confidence: Optional[float]
+
+class SegmentType(str, Enum):
+    CAPTION = "Caption"
+    FOOTNOTE = "Footnote"
+    FORMULA = "Formula"
+    LIST_ITEM = "ListItem"
+    PAGE = "Page"
+    PAGE_FOOTER = "PageFooter"
+    PAGE_HEADER = "PageHeader"
+    PICTURE = "Picture"
+    SECTION_HEADER = "SectionHeader"
+    TABLE = "Table"
+    TEXT = "Text"
+    TITLE = "Title"
+
+class Segment(BaseModel):
+    bbox: BoundingBox
+    content: str
+    page_height: float
+    html: Optional[str]
+    image: Optional[str]
+    markdown: Optional[str]
+    ocr: List[OCRResult]
+    page_number: int
+    page_width: float
+    segment_id: str
+    segment_type: SegmentType
+
+class Chunk(BaseModel):
+    chunk_id: str
+    chunk_length: int
+    segments: List[Segment]
+
+class ExtractedJson(BaseModel):
+    data: Dict
+
+class OutputResponse(BaseModel):
+    chunks: List[Chunk] = []
+    extracted_json: Optional[ExtractedJson]
+
+class Model(str, Enum):
+    FAST = "Fast"
+    HIGH_QUALITY = "HighQuality"
+
+class Configuration(BaseModel):
+    chunk_processing: ChunkProcessing = Field(default_factory=ChunkProcessing)
+    expires_in: Optional[int] = None
+    high_resolution: bool = False
+    json_schema: Optional[JsonSchema] = None
+    model: Optional[Model] = Field(None, deprecated=True)
+    ocr_strategy: OcrStrategy = OcrStrategy.AUTO
+    segment_processing: SegmentProcessing = Field(default_factory=SegmentProcessing)
+    segmentation_strategy: SegmentationStrategy = SegmentationStrategy.LAYOUT_ANALYSIS
+    target_chunk_length: Optional[int] = Field(None, deprecated=True)
+
+
+class Status(str, Enum):
+    STARTING = "Starting"
+    PROCESSING = "Processing"
+    SUCCEEDED = "Succeeded"
+    FAILED = "Failed"
+
+class TaskResponse(BaseModel, HeadersMixin):
+    configuration: Configuration
+    created_at: datetime
+    expires_at: Optional[datetime]
+    file_name: Optional[str]
+    finished_at: Optional[datetime]
+    input_file_url: Optional[str]
+    message: str
+    output: Optional[OutputResponse]
+    page_count: Optional[int]
+    pdf_url: Optional[str]
+    status: Status
+    task_id: str
+    task_url: Optional[str]
+    _api_key: Optional[str] = PrivateAttr(default=None)
+
+    def with_api_key(self, api_key: str) -> 'TaskResponse':
+        """Helper function to set api key on a TaskResponse after creation"""
+        self._api_key = api_key
+        return self
+
+    def poll(self) -> 'TaskResponse':
+        """Poll the task for completion"""
+        if not self.task_url:
+            raise ValueError("Task URL not found in response")
+        
+        while True:
+            r = requests.get(self.task_url, headers=self._headers())
+            r.raise_for_status()
+            self.__dict__.update(r.json())
+            if self.status == "Failed":
+                raise ValueError(self.message)
+            if self.status not in ("Starting", "Processing"):
+                return self
+            time.sleep(0.5)
+
+    async def poll_async(self) -> 'TaskResponse':
+        """Async poll the task for completion"""
+        if not self.task_url:
+            raise ValueError("Task URL not found in response")
+        
+        async with httpx.AsyncClient() as client:
+            while True:
+                r = await client.get(self.task_url, headers=self._headers())
+                r.raise_for_status()
+                self.__dict__.update(r.json())
+                if self.status == "Failed":
+                    raise ValueError(self.message)
+                if self.status not in ("Starting", "Processing"):
+                    return self
+                await asyncio.sleep(0.5)
+
+
+    def _get_content(self, content_type: str) -> str:
+        """Helper method to get either HTML, Markdown, or raw content."""
+        if not self.output:
+            return ""
+        parts = []
+        for c in self.output.chunks:
+            for s in c.segments:
+                content = getattr(s, content_type)
+                if content:
+                    parts.append(content)
+        return "\n".join(parts)
+
+    def html(self) -> str:
+        """Get full HTML for the task"""
+        return self._get_content("html")
+
+    def markdown(self) -> str:
+        """Get full markdown for the task"""
+        return self._get_content("markdown")
+        
+    def content(self) -> str:
+        """Get full text for the task"""
+        return self._get_content("content")
+
+class TaskPayload(BaseModel):
+    current_configuration: Configuration
+    file_name: str
+    image_folder_location: str
+    input_location: str
+    output_location: str
+    pdf_location: str
+    previous_configuration: Optional[Configuration]
+    task_id: str
+    user_id: str

chunkr_ai-0.0.2/src/chunkr_ai.egg-info/PKG-INFO

@@ -0,0 +1,16 @@
+Metadata-Version: 2.2
+Name: chunkr-ai
+Version: 0.0.2
+Summary: Python client for chunkr: open source document intelligence
+Author-email: Ishaan Kapoor <ishaan@lumina.sh>
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: build>=1.2.2.post1
+Requires-Dist: httpx>=0.28.1
+Requires-Dist: pillow>=11.1.0
+Requires-Dist: pydantic>=2.10.4
+Requires-Dist: python-dotenv>=1.0.1
+Requires-Dist: requests>=2.32.3
+Requires-Dist: twine>=6.0.1
+Provides-Extra: test
+Requires-Dist: pytest>=8.3.4; extra == "test"

chunkr_ai-0.0.2/src/chunkr_ai.egg-info/SOURCES.txt

@@ -0,0 +1,16 @@
+LICENSE
+README.md
+pyproject.toml
+src/chunkr_ai/__init__.py
+src/chunkr_ai/main.py
+src/chunkr_ai.egg-info/PKG-INFO
+src/chunkr_ai.egg-info/SOURCES.txt
+src/chunkr_ai.egg-info/dependency_links.txt
+src/chunkr_ai.egg-info/requires.txt
+src/chunkr_ai.egg-info/top_level.txt
+src/chunkr_ai/api/api.py
+src/chunkr_ai/api/auth.py
+src/chunkr_ai/api/chunkr.py
+src/chunkr_ai/api/chunkr_async.py
+src/chunkr_ai/api/models.py
+tests/test_chunkr.py

chunkr_ai-0.0.2/src/chunkr_ai.egg-info/requires.txt

@@ -0,0 +1,10 @@
+build>=1.2.2.post1
+httpx>=0.28.1
+pillow>=11.1.0
+pydantic>=2.10.4
+python-dotenv>=1.0.1
+requests>=2.32.3
+twine>=6.0.1
+
+[test]
+pytest>=8.3.4

chunkr_ai-0.0.2/tests/test_chunkr.py

@@ -0,0 +1,69 @@
+import pytest
+import os
+from pathlib import Path
+from PIL import Image
+import io
+from chunkr_ai import Chunkr, ChunkrAsync
+from chunkr_ai.api.models import TaskResponse
+
+# Test fixtures
+@pytest.fixture
+def chunkr():
+    return Chunkr()
+
+@pytest.fixture
+def async_chunkr():
+    return ChunkrAsync()
+
+@pytest.fixture
+def sample_pdf():
+    # Create a temporary PDF file for testing
+    content = b"%PDF-1.4 test content"
+    pdf_path = Path("test_document.pdf")
+    pdf_path.write_bytes(content)
+    yield str(pdf_path)
+    pdf_path.unlink()  # Cleanup after tests
+
+@pytest.fixture
+def sample_image():
+    # Create a test image
+    img = Image.new('RGB', (100, 100), color='red')
+    return img
+
+def test_prepare_file_string(chunkr, sample_pdf):
+    filename, file_obj = chunkr._prepare_file(sample_pdf)
+    assert filename == "test_document.pdf"
+    assert hasattr(file_obj, 'read')
+
+def test_prepare_file_image(chunkr, sample_image):
+    filename, file_obj = chunkr._prepare_file(sample_image)
+    assert filename == "image.png"
+    assert isinstance(file_obj, io.BytesIO)
+
+def test_prepare_file_bytes(chunkr):
+    test_bytes = b"test content"
+    filename, file_obj = chunkr._prepare_file(test_bytes)
+    assert filename == "document"
+    assert isinstance(file_obj, io.BytesIO)
+
+def test_send_file_string(chunkr, sample_pdf):
+    response = chunkr.upload(sample_pdf)
+    
+    assert isinstance(response, TaskResponse)
+    assert response.task_id is not None
+    assert response.status in ["pending", "processing", "completed"]
+
+def test_send_file_image(chunkr, sample_image):
+    response = chunkr.upload(sample_image)
+    
+    assert isinstance(response, TaskResponse)
+    assert response.task_id is not None
+    assert response.status in ["pending", "processing", "completed"]
+
+def test_send_file_bytes(chunkr):
+    test_bytes = b"This is a test document content"
+    response = chunkr.upload(test_bytes)
+    
+    assert isinstance(response, TaskResponse)
+    assert response.task_id is not None
+    assert response.status in ["pending", "processing", "completed"]

chunkr_ai-0.0.1/PKG-INFO

@@ -1,7 +0,0 @@
-Metadata-Version: 2.1
-Name: chunkr-ai
-Version: 0.0.1
-Summary: PDF chunking
-Author-email: Ishaan Kapoor <ishaan@lumina.sh>
-Description-Content-Type: text/markdown
-License-File: LICENSE

chunkr_ai-0.0.1/pyproject.toml

@@ -1,11 +0,0 @@
-[build-system]
-requires = ["setuptools>=42", "wheel"]
-build-backend = "setuptools.build_meta"
-
-[project]
-name = "chunkr-ai"
-version = "0.0.1"
-authors = [{"name" = "Ishaan Kapoor", "email" = "ishaan@lumina.sh"}]
-description = "PDF chunking"
-readme = "README.md"
-license = {"file" = "LICENSE"}

chunkr_ai-0.0.1/src/chunkr_ai.egg-info/PKG-INFO

@@ -1,7 +0,0 @@
-Metadata-Version: 2.1
-Name: chunkr-ai
-Version: 0.0.1
-Summary: PDF chunking
-Author-email: Ishaan Kapoor <ishaan@lumina.sh>
-Description-Content-Type: text/markdown
-License-File: LICENSE

chunkr_ai-0.0.1/src/chunkr_ai.egg-info/SOURCES.txt

@@ -1,9 +0,0 @@
-LICENSE
-README.md
-pyproject.toml
-src/chunkr_ai/__init__.py
-src/chunkr_ai/main.py
-src/chunkr_ai.egg-info/PKG-INFO
-src/chunkr_ai.egg-info/SOURCES.txt
-src/chunkr_ai.egg-info/dependency_links.txt
-src/chunkr_ai.egg-info/top_level.txt