chunkr-ai 0.0.43__py3-none-any.whl → 0.0.44__py3-none-any.whl

chunkr_ai/api/configuration.py

@@ -1,6 +1,7 @@
 from pydantic import BaseModel, Field, ConfigDict
 from enum import Enum
-from typing import Any, List, Optional
+from typing import Any, List, Optional, Union
+from pydantic import field_validator
 
 class GenerationStrategy(str, Enum):
     LLM = "LLM"
@@ -10,11 +11,18 @@ class CroppingStrategy(str, Enum):
     ALL = "All"
     AUTO = "Auto"
 
+class EmbedSource(str, Enum):
+    HTML = "HTML"
+    MARKDOWN = "Markdown"
+    LLM = "LLM"
+    CONTENT = "Content"
+
 class GenerationConfig(BaseModel):
     html: Optional[GenerationStrategy] = None
     llm: Optional[str] = None
     markdown: Optional[GenerationStrategy] = None
     crop_image: Optional[CroppingStrategy] = None
+    embed_sources: Optional[List[EmbedSource]] = Field(default_factory=lambda: [EmbedSource.MARKDOWN])
 
 class SegmentProcessing(BaseModel):
     model_config = ConfigDict(populate_by_name=True, alias_generator=str.title)
@@ -32,9 +40,83 @@ class SegmentProcessing(BaseModel):
     text: Optional[GenerationConfig] = Field(default=None, alias="Text")
     title: Optional[GenerationConfig] = Field(default=None, alias="Title")
 
+class Tokenizer(str, Enum):
+    WORD = "Word"
+    CL100K_BASE = "Cl100kBase"
+    XLM_ROBERTA_BASE = "XlmRobertaBase"
+    BERT_BASE_UNCASED = "BertBaseUncased"
+
+class TokenizerType(BaseModel):
+    enum_value: Optional[Tokenizer] = None
+    string_value: Optional[str] = None
+
+    @classmethod
+    def from_enum(cls, enum_value: Tokenizer) -> "TokenizerType":
+        return cls(enum_value=enum_value)
+    
+    @classmethod
+    def from_string(cls, string_value: str) -> "TokenizerType":
+        return cls(string_value=string_value)
+    
+    def __str__(self) -> str:
+        if self.enum_value is not None:
+            return f"enum:{self.enum_value.value}"
+        elif self.string_value is not None:
+            return f"string:{self.string_value}"
+        return ""
+    
+    model_config = ConfigDict(
+        json_encoders={
+            'TokenizerType': lambda v: v.model_dump()
+        }
+    )
+    
+    def model_dump(self, **kwargs):
+        if self.enum_value is not None:
+            return {"Enum": self.enum_value.value}
+        elif self.string_value is not None:
+            return {"String": self.string_value}
+        return {}
+
 class ChunkProcessing(BaseModel):
-    ignore_headers_and_footers: Optional[bool] = None
+    ignore_headers_and_footers: Optional[bool] = True
     target_length: Optional[int] = None
+    tokenizer: Optional[Union[TokenizerType, Tokenizer, str]] = None
+
+    model_config = ConfigDict(
+        arbitrary_types_allowed=True,
+        json_encoders={
+            TokenizerType: lambda v: v.model_dump()
+        }
+    )
+
+    @field_validator('tokenizer', mode='before')
+    def validate_tokenizer(cls, v):
+        if v is None:
+            return None
+        
+        if isinstance(v, TokenizerType):
+            return v
+        
+        if isinstance(v, Tokenizer):
+            return TokenizerType(enum_value=v)
+        
+        if isinstance(v, dict):
+            if "Enum" in v:
+                try:
+                    return TokenizerType(enum_value=Tokenizer(v["Enum"]))
+                except ValueError:
+                    return TokenizerType(string_value=v["Enum"])
+            elif "String" in v:
+                return TokenizerType(string_value=v["String"])
+            
+        if isinstance(v, str):
+            try:
+                return TokenizerType(enum_value=Tokenizer(v))
+            except ValueError:
+                return TokenizerType(string_value=v)
+                
+        raise ValueError(f"Cannot convert {v} to TokenizerType")
 
 class OcrStrategy(str, Enum):
     ALL = "All"

chunkr_ai/models.py

@@ -4,6 +4,7 @@ from .api.configuration import (
     ChunkProcessing,
     Configuration,
     CroppingStrategy,
+    EmbedSource,
     GenerationStrategy,
     GenerationConfig,
     Model,
@@ -16,6 +17,8 @@ from .api.configuration import (
     SegmentationStrategy,
     Status,
     Pipeline,
+    Tokenizer,
+    TokenizerType,
 )
 from .api.task_response import TaskResponse
 
@@ -25,6 +28,7 @@ __all__ = [
     "ChunkProcessing",
     "Configuration",
     "CroppingStrategy",
+    "EmbedSource",
     "GenerationConfig",
     "GenerationStrategy",
     "Model",
@@ -38,5 +42,6 @@ __all__ = [
     "Status",
     "TaskResponse",
     "Pipeline",
+    "Tokenizer",
+    "TokenizerType",
 ]
-

{chunkr_ai-0.0.43.dist-info → chunkr_ai-0.0.44.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: chunkr-ai
-Version: 0.0.43
+Version: 0.0.44
 Summary: Python client for Chunkr: open source document intelligence
 Author-email: Ishaan Kapoor <ishaan@lumina.sh>
 License: MIT License

{chunkr_ai-0.0.43.dist-info → chunkr_ai-0.0.44.dist-info}/RECORD

@@ -1,16 +1,16 @@
 chunkr_ai/__init__.py,sha256=6KpYv2lmD6S5z2kc9pqwuLP5VDHmOuu2qDZArUIhb1s,53
-chunkr_ai/models.py,sha256=tOI7ylkhyeFfCLMisk96EPsH4UEcjBx1Mcisxc_AYXI,757
+chunkr_ai/models.py,sha256=Pfr8S0qbC5GSgI3zCE63bnBCWOOjiExBXIrSRPyLhkc,864
 chunkr_ai/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 chunkr_ai/api/auth.py,sha256=hlv0GiUmlsbFO1wLL9sslqOnsBSoBqkL_6Mk2SDvxgE,413
 chunkr_ai/api/chunkr.py,sha256=BzwcKNCuLfVR-HzgY8tKStsW4pIDVVjBgnEqPLyUUMM,3292
 chunkr_ai/api/chunkr_base.py,sha256=FDl0Ew8eOY4hur5FFqPENZiq9YQy0G3XWEqcKPeCO-U,6130
-chunkr_ai/api/configuration.py,sha256=Kkqxco8M-xgijUMsmtL8rJpMxrnSEnsQY8dUP8Cg5oc,3947
+chunkr_ai/api/configuration.py,sha256=jHEAz3H9uRh22jpSqnGyCdT4VbkCE_L_1fm0uVlv_1U,6527
 chunkr_ai/api/decorators.py,sha256=VJX4qGBIL00K2zY8bh5KAMWv7SltJ38TvPJH06FnFss,4415
 chunkr_ai/api/misc.py,sha256=QN-2YWQ8e3VvvK63Ua-e8jsx6gxVxkO88Z96yWOofu0,3653
 chunkr_ai/api/protocol.py,sha256=LjPrYSq52m1afIlAo0yVGXlGZxPRh8J6g7S4PAit3Zo,388
 chunkr_ai/api/task_response.py,sha256=6kk9g2f7OZB3PAsmp4Or5A42r1dXTAzWAHEIVtLQ9sA,6545
-chunkr_ai-0.0.43.dist-info/licenses/LICENSE,sha256=w3R12yNDyZpMiy2lxy_hvNbsldC75ww79sF0u11rkho,1069
-chunkr_ai-0.0.43.dist-info/METADATA,sha256=EUBjfees5n8KRCpn65HFwsvzmSZYJ7wl5rkkbADwkd0,7053
-chunkr_ai-0.0.43.dist-info/WHEEL,sha256=1tXe9gY0PYatrMPMDd6jXqjfpz_B-Wqm32CPfRC58XU,91
-chunkr_ai-0.0.43.dist-info/top_level.txt,sha256=0IZY7PZIiS8bw5r4NUQRUQ-ATi-L_3vLQVq3ZLouOW8,10
-chunkr_ai-0.0.43.dist-info/RECORD,,
+chunkr_ai-0.0.44.dist-info/licenses/LICENSE,sha256=w3R12yNDyZpMiy2lxy_hvNbsldC75ww79sF0u11rkho,1069
+chunkr_ai-0.0.44.dist-info/METADATA,sha256=Gk-DiU78MVBXlYk3BjcKCVGfy1JL-SB0wj8p-ooovYs,7053
+chunkr_ai-0.0.44.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+chunkr_ai-0.0.44.dist-info/top_level.txt,sha256=0IZY7PZIiS8bw5r4NUQRUQ-ATi-L_3vLQVq3ZLouOW8,10
+chunkr_ai-0.0.44.dist-info/RECORD,,

{chunkr_ai-0.0.43.dist-info → chunkr_ai-0.0.44.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (77.0.3)
+Generator: setuptools (78.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any