docler 0.1.2__tar.gz → 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. {docler-0.1.2 → docler-0.2.0}/.pre-commit-config.yaml +1 -1
  2. {docler-0.1.2 → docler-0.2.0}/PKG-INFO +22 -6
  3. {docler-0.1.2 → docler-0.2.0}/pyproject.toml +22 -4
  4. docler-0.2.0/src/docler/__init__.py +34 -0
  5. docler-0.2.0/src/docler/__main__.py +55 -0
  6. docler-0.2.0/src/docler/chunkers/ai_chunker/__init__.py +5 -0
  7. docler-0.2.0/src/docler/chunkers/ai_chunker/chunker.py +161 -0
  8. {docler-0.1.2 → docler-0.2.0}/src/docler/chunkers/base.py +5 -12
  9. docler-0.2.0/src/docler/chunkers/llamaindex_chunker/__init__.py +5 -0
  10. docler-0.2.0/src/docler/chunkers/llamaindex_chunker/chunker.py +167 -0
  11. docler-0.2.0/src/docler/chunkers/markdown_chunker/__init__.py +5 -0
  12. docler-0.2.0/src/docler/chunkers/markdown_chunker/chunker.py +152 -0
  13. docler-0.2.0/src/docler/common_types.py +37 -0
  14. docler-0.2.0/src/docler/configs/chunker_configs.py +84 -0
  15. docler-0.1.2/src/docler/configs.py → docler-0.2.0/src/docler/configs/converter_configs.py +21 -16
  16. docler-0.2.0/src/docler/configs/vector_db_configs.py +100 -0
  17. docler-0.2.0/src/docler/converters/__init__.py +0 -0
  18. docler-0.2.0/src/docler/converters/azure_provider/__init__.py +0 -0
  19. docler-0.2.0/src/docler/converters/azure_provider/converter.py +237 -0
  20. {docler-0.1.2/src/docler → docler-0.2.0/src/docler/converters}/base.py +6 -6
  21. docler-0.2.0/src/docler/converters/datalab_provider/__init__.py +5 -0
  22. {docler-0.1.2/src/docler → docler-0.2.0/src/docler/converters}/datalab_provider/provider.py +14 -30
  23. {docler-0.1.2/src/docler → docler-0.2.0/src/docler/converters}/dir_converter.py +3 -8
  24. docler-0.2.0/src/docler/converters/docling_provider/__init__.py +5 -0
  25. {docler-0.1.2/src/docler → docler-0.2.0/src/docler/converters}/docling_provider/provider.py +56 -56
  26. docler-0.1.2/src/docler/lang_code.py → docler-0.2.0/src/docler/converters/docling_provider/utils.py +17 -34
  27. docler-0.2.0/src/docler/converters/kreuzberg_provider/__init__.py +5 -0
  28. {docler-0.1.2/src/docler → docler-0.2.0/src/docler/converters}/kreuzberg_provider/provider.py +6 -8
  29. docler-0.2.0/src/docler/converters/llamaindex_provider/__init__.py +5 -0
  30. docler-0.2.0/src/docler/converters/llamaindex_provider/provider.py +124 -0
  31. docler-0.2.0/src/docler/converters/llamaparse_provider/__init__.py +5 -0
  32. docler-0.2.0/src/docler/converters/llamaparse_provider/provider.py +151 -0
  33. docler-0.2.0/src/docler/converters/llm_provider/__init__.py +5 -0
  34. {docler-0.1.2/src/docler/litellm_provider → docler-0.2.0/src/docler/converters/llm_provider}/provider.py +27 -51
  35. docler-0.2.0/src/docler/converters/marker_provider/__init__.py +5 -0
  36. {docler-0.1.2/src/docler → docler-0.2.0/src/docler/converters}/marker_provider/provider.py +18 -23
  37. docler-0.2.0/src/docler/converters/markitdown_provider/__init__.py +5 -0
  38. {docler-0.1.2/src/docler → docler-0.2.0/src/docler/converters}/markitdown_provider/provider.py +5 -12
  39. docler-0.2.0/src/docler/converters/mistral_provider/__init__.py +5 -0
  40. {docler-0.1.2/src/docler → docler-0.2.0/src/docler/converters}/mistral_provider/provider.py +10 -27
  41. {docler-0.1.2/src/docler → docler-0.2.0/src/docler/converters}/registry.py +6 -7
  42. docler-0.2.0/src/docler/embeddings/__init__.py +1 -0
  43. docler-0.2.0/src/docler/embeddings/base.py +63 -0
  44. docler-0.2.0/src/docler/embeddings/bge_provider.py +47 -0
  45. docler-0.2.0/src/docler/embeddings/litellm_provider.py +109 -0
  46. docler-0.2.0/src/docler/embeddings/openai_provider.py +106 -0
  47. docler-0.2.0/src/docler/embeddings/stf_provider.py +51 -0
  48. docler-0.2.0/src/docler/provider.py +25 -0
  49. docler-0.2.0/src/docler/py.typed +0 -0
  50. docler-0.2.0/src/docler/resources/pdf_sample.pdf +0 -0
  51. docler-0.2.0/src/docler/streamlit_app/__init__.py +5 -0
  52. docler-0.2.0/src/docler/streamlit_app/app.py +58 -0
  53. docler-0.2.0/src/docler/streamlit_app/chunkers.py +15 -0
  54. docler-0.2.0/src/docler/streamlit_app/converters.py +23 -0
  55. docler-0.2.0/src/docler/streamlit_app/state.py +53 -0
  56. docler-0.2.0/src/docler/streamlit_app/step1_conversion.py +108 -0
  57. docler-0.2.0/src/docler/streamlit_app/step2_chunking.py +201 -0
  58. docler-0.2.0/src/docler/streamlit_app/utils.py +36 -0
  59. docler-0.2.0/src/docler/streamlit_app.py +145 -0
  60. docler-0.2.0/src/docler/streamlit_chunk_app.py +127 -0
  61. docler-0.2.0/src/docler/utils.py +74 -0
  62. docler-0.2.0/src/docler/vector_db/__init__.py +1 -0
  63. docler-0.2.0/src/docler/vector_db/base.py +447 -0
  64. docler-0.2.0/src/docler/vector_db/dbs/__init__.py +0 -0
  65. docler-0.2.0/src/docler/vector_db/dbs/chroma_db/__init__.py +5 -0
  66. docler-0.2.0/src/docler/vector_db/dbs/chroma_db/db.py +259 -0
  67. docler-0.2.0/src/docler/vector_db/dbs/openai_db/__init__.py +0 -0
  68. docler-0.2.0/src/docler/vector_db/dbs/openai_db/db.py +327 -0
  69. docler-0.2.0/src/docler/vector_db/dbs/openai_db/manager.py +301 -0
  70. docler-0.2.0/src/docler/vector_db/dbs/qdrant_db/__init__.py +5 -0
  71. docler-0.2.0/src/docler/vector_db/dbs/qdrant_db/db.py +334 -0
  72. docler-0.2.0/src/docler/vector_db/dbs/upstash_db/__init__.py +7 -0
  73. docler-0.2.0/src/docler/vector_db/dbs/upstash_db/db.py +426 -0
  74. docler-0.2.0/src/docler/vector_db/dbs/upstash_db/manager.py +298 -0
  75. docler-0.2.0/tests/resources/pdf_sample.pdf +0 -0
  76. docler-0.2.0/tests/test_chunkers.py +156 -0
  77. docler-0.1.2/src/docler/__init__.py +0 -34
  78. docler-0.1.2/src/docler/__main__.py +0 -81
  79. docler-0.1.2/src/docler/common_types.py +0 -6
  80. docler-0.1.2/src/docler/datalab_provider/__init__.py +0 -5
  81. docler-0.1.2/src/docler/docling_provider/__init__.py +0 -5
  82. docler-0.1.2/src/docler/kreuzberg_provider/__init__.py +0 -5
  83. docler-0.1.2/src/docler/litellm_provider/__init__.py +0 -5
  84. docler-0.1.2/src/docler/marker_provider/__init__.py +0 -5
  85. docler-0.1.2/src/docler/markitdown_provider/__init__.py +0 -5
  86. docler-0.1.2/src/docler/mistral_provider/__init__.py +0 -5
  87. docler-0.1.2/src/docler/olmocr_provider/__init__.py +0 -5
  88. docler-0.1.2/src/docler/olmocr_provider/provider.py +0 -196
  89. docler-0.1.2/src/docler/streamlit_app.py +0 -103
  90. docler-0.1.2/src/docler/utils.py +0 -21
  91. {docler-0.1.2 → docler-0.2.0}/.copier-answers.yml +0 -0
  92. {docler-0.1.2 → docler-0.2.0}/.github/FUNDING.yml +0 -0
  93. {docler-0.1.2 → docler-0.2.0}/.github/copilot-instructions.md +0 -0
  94. {docler-0.1.2 → docler-0.2.0}/.github/dependabot.yml +0 -0
  95. {docler-0.1.2 → docler-0.2.0}/.github/workflows/build.yml +0 -0
  96. {docler-0.1.2 → docler-0.2.0}/.github/workflows/documentation.yml +0 -0
  97. {docler-0.1.2 → docler-0.2.0}/.gitignore +0 -0
  98. {docler-0.1.2 → docler-0.2.0}/LICENSE +0 -0
  99. {docler-0.1.2 → docler-0.2.0}/README.md +0 -0
  100. {docler-0.1.2 → docler-0.2.0}/docs/.empty +0 -0
  101. {docler-0.1.2 → docler-0.2.0}/duties.py +0 -0
  102. {docler-0.1.2 → docler-0.2.0}/mkdocs.yml +0 -0
  103. {docler-0.1.2 → docler-0.2.0}/overrides/_dummy.txt +0 -0
  104. {docler-0.1.2 → docler-0.2.0}/src/docler/chunkers/__init__.py +0 -0
  105. /docler-0.1.2/src/docler/py.typed → /docler-0.2.0/src/docler/configs/__init__.py +0 -0
  106. {docler-0.1.2 → docler-0.2.0}/src/docler/models.py +0 -0
  107. {docler-0.1.2 → docler-0.2.0}/tests/__init__.py +0 -0
  108. {docler-0.1.2 → docler-0.2.0}/tests/conftest.py +0 -0
  109. {docler-0.1.2 → docler-0.2.0}/tests/test_converters.py +0 -0
@@ -35,7 +35,7 @@ repos:
35
35
  additional_dependencies: [orjson, pydantic]
36
36
 
37
37
  - repo: https://github.com/charliermarsh/ruff-pre-commit
38
- rev: v0.9.10
38
+ rev: v0.11.0
39
39
  hooks:
40
40
  - id: ruff
41
41
  - id: ruff-format
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: docler
3
- Version: 0.1.2
3
+ Version: 0.2.0
4
4
  Summary: Abstractions & Tools for OCR / document processing
5
5
  Project-URL: Documentation, https://phil65.github.io/docler/
6
6
  Project-URL: Source, https://github.com/phil65/docler
@@ -48,13 +48,20 @@ Requires-Dist: anyenv>=0.4.1
48
48
  Requires-Dist: pydantic
49
49
  Requires-Dist: upathtools>=0.4.3
50
50
  Provides-Extra: all
51
+ Requires-Dist: chromadb; extra == 'all'
51
52
  Requires-Dist: docling; extra == 'all'
52
53
  Requires-Dist: kreuzberg; extra == 'all'
53
- Requires-Dist: litellm; extra == 'all'
54
- Requires-Dist: marker; extra == 'all'
55
- Requires-Dist: markitdown[all]>=0.1.0a1; extra == 'all'
54
+ Requires-Dist: llama-index-readers-smart-pdf-loader; extra == 'all'
55
+ Requires-Dist: llmling-agent[pydantic-ai]; extra == 'all'
56
+ Requires-Dist: llmsherpa; extra == 'all'
57
+ Requires-Dist: marker-pdf; extra == 'all'
58
+ Requires-Dist: markitdown; extra == 'all'
56
59
  Requires-Dist: mistralai; extra == 'all'
60
+ Requires-Dist: openai; extra == 'all'
57
61
  Requires-Dist: streamlit; extra == 'all'
62
+ Requires-Dist: upstash-vector; extra == 'all'
63
+ Provides-Extra: chromadb
64
+ Requires-Dist: chromadb>=0.6.3; extra == 'chromadb'
58
65
  Provides-Extra: docling
59
66
  Requires-Dist: docling; extra == 'docling'
60
67
  Provides-Extra: kreuzberg
@@ -62,14 +69,23 @@ Requires-Dist: kreuzberg; extra == 'kreuzberg'
62
69
  Provides-Extra: litellm
63
70
  Requires-Dist: litellm; extra == 'litellm'
64
71
  Requires-Dist: tokonomics; extra == 'litellm'
72
+ Provides-Extra: llama-index
73
+ Requires-Dist: llama-index; extra == 'llama-index'
74
+ Provides-Extra: llama-parse
75
+ Requires-Dist: llama-parse; extra == 'llama-parse'
65
76
  Provides-Extra: marker
66
77
  Requires-Dist: marker-pdf; extra == 'marker'
67
- Provides-Extra: markitdown
68
- Requires-Dist: markitdown[all]>=0.1.0a1; extra == 'markitdown'
69
78
  Provides-Extra: mistralai
70
79
  Requires-Dist: mistralai; extra == 'mistralai'
80
+ Provides-Extra: openai
81
+ Requires-Dist: openai; extra == 'openai'
82
+ Provides-Extra: smart-pdf
83
+ Requires-Dist: llama-index-readers-smart-pdf-loader; extra == 'smart-pdf'
84
+ Requires-Dist: llmsherpa; extra == 'smart-pdf'
71
85
  Provides-Extra: streamlit
72
86
  Requires-Dist: streamlit; extra == 'streamlit'
87
+ Provides-Extra: upstash
88
+ Requires-Dist: upstash-vector; extra == 'upstash'
73
89
  Description-Content-Type: text/markdown
74
90
 
75
91
  # Docler
@@ -45,19 +45,34 @@ marker = ["marker-pdf"]
45
45
  mistralai = ["mistralai"]
46
46
  docling = ["docling"]
47
47
  # olmocr = ["olmocr", "accelerate"]
48
- markitdown = ["markitdown[all]>=0.1.0a1"]
48
+ # markitdown = ["markitdown[all]"]
49
49
  kreuzberg = ["kreuzberg"]
50
50
  litellm = ["litellm", "tokonomics"]
51
51
  streamlit = ["streamlit"]
52
+ llama-index = ["llama-index"]
53
+ llama-parse = ["llama-parse"]
54
+ smart-pdf = ["llmsherpa", "llama-index-readers-smart-pdf-loader"]
55
+ chromadb = ["chromadb>=0.6.3"]
56
+ upstash = ["upstash-vector"]
57
+ openai = ["openai"]
52
58
  all = [
53
- "marker",
59
+ "marker-pdf",
54
60
  "mistralai",
55
61
  "docling",
56
- "markitdown[all]>=0.1.0a1",
62
+ "markitdown",
57
63
  "kreuzberg",
58
- "litellm",
64
+ # "litellm",
65
+ "llmling-agent[pydantic-ai]",
66
+ # "llama-parse",
59
67
  "streamlit",
68
+ "llmsherpa",
69
+ "llama-index-readers-smart-pdf-loader",
70
+ "chromadb",
71
+ "upstash-vector",
72
+ "openai",
60
73
  ]
74
+
75
+
61
76
  [project.scripts]
62
77
  docler = "docler.__main__:cli"
63
78
 
@@ -70,6 +85,7 @@ dev = [
70
85
  "pytest-cov",
71
86
  "pyreadline3",
72
87
  "devtools",
88
+ "pytest-asyncio",
73
89
  # Only add below (Copier)
74
90
  ]
75
91
  benchmark = ["pyinstrument"]
@@ -97,6 +113,7 @@ testpaths = "tests/"
97
113
  log_cli = true
98
114
  log_format = "%(asctime)s %(levelname)s %(message)s"
99
115
  log_date_format = "%Y-%m-%d %H:%M:%S"
116
+ asyncio_mode = "auto"
100
117
 
101
118
  [tool.coverage.report]
102
119
  exclude_lines = [
@@ -215,6 +232,7 @@ ignore = [
215
232
  # "PLR2004", # Magic values instead of named consts
216
233
  "SLF001", # Private member accessed
217
234
  "TRY003", # Avoid specifying long messages outside the exception class
235
+ "TC006",
218
236
  ]
219
237
 
220
238
  [tool.ruff.lint.flake8-quotes]
@@ -0,0 +1,34 @@
1
+ """Document conversion library supporting multiple providers."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from docler.converters.base import DocumentConverter
6
+ from docler.converters.dir_converter import Conversion, DirectoryConverter
7
+ from docler.models import Document, Image, ImageReferenceFormat
8
+ from docler.converters.registry import ConverterRegistry
9
+
10
+ # Import providers
11
+ from docler.converters.docling_provider import DoclingConverter
12
+ from docler.converters.marker_provider import MarkerConverter
13
+ from docler.converters.mistral_provider import MistralConverter
14
+ from docler.converters.llm_provider import LLMConverter
15
+ from docler.converters.datalab_provider import DataLabConverter
16
+ from docler.converters.llamaparse_provider import LlamaParseConverter
17
+
18
+ __version__ = "0.2.0"
19
+
20
+ __all__ = [
21
+ "Conversion",
22
+ "ConverterRegistry",
23
+ "DataLabConverter",
24
+ "DirectoryConverter",
25
+ "DoclingConverter",
26
+ "Document",
27
+ "DocumentConverter",
28
+ "Image",
29
+ "ImageReferenceFormat",
30
+ "LLMConverter",
31
+ "LlamaParseConverter",
32
+ "MarkerConverter",
33
+ "MistralConverter",
34
+ ]
@@ -0,0 +1,55 @@
1
+ """Command line interface for Docler document converter."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import logging
6
+ import os
7
+ import pathlib
8
+ import subprocess
9
+ import sys
10
+
11
+ import typer
12
+
13
+
14
+ cli = typer.Typer(help="Docler document converter CLI", no_args_is_help=True)
15
+
16
+ logger = logging.getLogger(__name__)
17
+
18
+ PACKAGE_DIR = pathlib.Path(__file__).parent
19
+
20
+
21
+ @cli.command()
22
+ def serve():
23
+ """Start the Streamlit web interface."""
24
+ app_path = str(PACKAGE_DIR / "streamlit_app" / "app.py")
25
+ try:
26
+ cmd = [sys.executable, "-m", "streamlit", "run", app_path]
27
+ subprocess.run(cmd, env=os.environ.copy(), check=True)
28
+ except subprocess.CalledProcessError as e:
29
+ # msg = f"Failed to start Streamlit: {e}"
30
+ raise typer.Exit(1) from e
31
+ except KeyboardInterrupt:
32
+ logger.info("Shutting down...")
33
+
34
+
35
+ @cli.command("chunk_ui")
36
+ def chunk_ui():
37
+ """Start the Streamlit web interface."""
38
+ app_path = str(PACKAGE_DIR / "streamlit_chunk_app.py")
39
+ try:
40
+ cmd = [sys.executable, "-m", "streamlit", "run", app_path]
41
+ subprocess.run(cmd, env=os.environ.copy(), check=True)
42
+ except subprocess.CalledProcessError as e:
43
+ # msg = f"Failed to start Streamlit: {e}"
44
+ raise typer.Exit(1) from e
45
+ except KeyboardInterrupt:
46
+ logger.info("Shutting down...")
47
+
48
+
49
+ def main():
50
+ """Main entry point."""
51
+ cli()
52
+
53
+
54
+ if __name__ == "__main__":
55
+ main()
@@ -0,0 +1,5 @@
1
+ """AI Chunker."""
2
+
3
+ from docler.chunkers.ai_chunker.chunker import AIChunker
4
+
5
+ __all__ = ["AIChunker"]
@@ -0,0 +1,161 @@
1
+ """AI-based markdown chunking implementation."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import TYPE_CHECKING, Any, ClassVar
6
+
7
+ from pydantic import BaseModel, ConfigDict
8
+
9
+ from docler.chunkers.base import TextChunk, TextChunker
10
+
11
+
12
+ if TYPE_CHECKING:
13
+ from llmling_agent.agent.agent import AgentType
14
+
15
+ from docler.models import Document
16
+
17
+
18
+ class Chunk(BaseModel):
19
+ """A chunk of text with semantic metadata."""
20
+
21
+ start_row: int
22
+ """Start line number (1-based)"""
23
+
24
+ end_row: int
25
+ """End line number (1-based)"""
26
+
27
+ keywords: list[str]
28
+ """Key terms and concepts in this chunk"""
29
+
30
+ references: list[int]
31
+ """Line numbers that this chunk references or depends on"""
32
+
33
+ model_config = ConfigDict(use_attribute_docstrings=True)
34
+
35
+
36
+ class Chunks(BaseModel):
37
+ """Collection of chunks with their metadata."""
38
+
39
+ chunks: list[Chunk]
40
+ """A list of chunks to extract from the document."""
41
+
42
+
43
+ SYS_PROMPT = """
44
+ You are an expert at dividing text into meaningful chunks while preserving context and relationships.
45
+
46
+ Analyze this text and split it into coherent chunks. For each chunk:
47
+ 1. Define its start and end line numbers (1-based)
48
+ 2. Extract key terms and concepts as keywords
49
+ 3. Note any line numbers it references or depends on
50
+
51
+ Rules:
52
+ - Each chunk should be semantically complete
53
+ - Identify cross-references between different parts of the text
54
+ - Keywords should be specific and relevant
55
+ - Line numbers must be accurate
56
+
57
+ """ # noqa: E501
58
+
59
+ CHUNKING_PROMPT = """
60
+ Here's the text with line numbers:
61
+
62
+ {numbered_text}
63
+
64
+ """
65
+
66
+
67
+ class AIChunker(TextChunker):
68
+ """LLM-based document chunker."""
69
+
70
+ REQUIRED_PACKAGES: ClassVar = {"llmling-agent"}
71
+
72
+ def __init__(
73
+ self,
74
+ model: str = "openrouter:openai/o3-mini", # google/gemini-2.0-flash-lite-001
75
+ provider: AgentType = "pydantic_ai",
76
+ min_chunk_size: int = 200,
77
+ max_chunk_size: int = 1500,
78
+ ):
79
+ """Initialize the AI chunker.
80
+
81
+ Args:
82
+ model: LLM model to use
83
+ provider: LLM provider to use
84
+ min_chunk_size: Minimum characters per chunk
85
+ max_chunk_size: Maximum characters per chunk
86
+ """
87
+ self.model = model
88
+ self.provider = provider
89
+ self.min_chunk_size = min_chunk_size
90
+ self.max_chunk_size = max_chunk_size
91
+
92
+ def _add_line_numbers(self, text: str) -> str:
93
+ """Add line numbers to text."""
94
+ lines = text.splitlines()
95
+ return "\n".join(f"{i + 1:3d} | {line}" for i, line in enumerate(lines))
96
+
97
+ async def _get_chunks(self, text: str) -> Chunks:
98
+ """Get chunk definitions from LLM."""
99
+ import llmling_agent
100
+
101
+ numbered_text = self._add_line_numbers(text)
102
+ agent: llmling_agent.StructuredAgent[None, Chunks] = llmling_agent.Agent(
103
+ model=self.model,
104
+ provider=self.provider, # pyright: ignore
105
+ system_prompt=SYS_PROMPT,
106
+ ).to_structured(Chunks)
107
+ prompt = CHUNKING_PROMPT.format(numbered_text=numbered_text)
108
+ response = await agent.run(prompt)
109
+ return response.content
110
+
111
+ def _create_text_chunk(
112
+ self,
113
+ doc: Document,
114
+ chunk: Chunk,
115
+ chunk_idx: int,
116
+ extra_metadata: dict[str, Any] | None = None,
117
+ ) -> TextChunk:
118
+ """Create a TextChunk from chunk definition."""
119
+ lines = doc.content.splitlines()
120
+ chunk_lines = lines[chunk.start_row - 1 : chunk.end_row]
121
+ chunk_text = "\n".join(chunk_lines)
122
+ base = extra_metadata or {}
123
+ metadata = {**base, "keywords": chunk.keywords, "references": chunk.references}
124
+ chunk_images = [i for i in doc.images if i.filename and i.filename in chunk_text]
125
+ return TextChunk(
126
+ text=chunk_text,
127
+ source_doc_id=doc.source_path or "",
128
+ chunk_index=chunk_idx,
129
+ images=chunk_images,
130
+ metadata=metadata,
131
+ )
132
+
133
+ async def split(
134
+ self,
135
+ doc: Document,
136
+ extra_metadata: dict[str, Any] | None = None,
137
+ ) -> list[TextChunk]:
138
+ """Split document into chunks using LLM analysis."""
139
+ # Get chunk definitions from LLM
140
+ chunks = await self._get_chunks(doc.content)
141
+
142
+ # Convert to TextChunks
143
+ return [
144
+ self._create_text_chunk(doc, chunk, i, extra_metadata)
145
+ for i, chunk in enumerate(chunks.chunks)
146
+ ]
147
+
148
+
149
+ if __name__ == "__main__":
150
+ import asyncio
151
+
152
+ from docler.models import Document
153
+
154
+ async def main():
155
+ # Example usage
156
+ doc = Document(source_path="example.txt", content=SYS_PROMPT)
157
+ chunker = AIChunker()
158
+ chunks = await chunker.split(doc)
159
+ print(chunks)
160
+
161
+ asyncio.run(main())
@@ -6,6 +6,8 @@ from abc import ABC, abstractmethod
6
6
  from dataclasses import dataclass, field
7
7
  from typing import TYPE_CHECKING, Any
8
8
 
9
+ from docler.provider import BaseProvider
10
+
9
11
 
10
12
  if TYPE_CHECKING:
11
13
  from collections.abc import Iterable
@@ -25,14 +27,10 @@ class TextChunk:
25
27
  metadata: dict[str, Any] = field(default_factory=dict)
26
28
 
27
29
 
28
- class TextChunker(ABC):
30
+ class TextChunker(BaseProvider, ABC):
29
31
  """Base class for text chunkers."""
30
32
 
31
- def __init__(
32
- self,
33
- chunk_size: int = 500,
34
- chunk_overlap: int = 50,
35
- ) -> None:
33
+ def __init__(self, chunk_size: int = 500, chunk_overlap: int = 50):
36
34
  self.chunk_size = chunk_size
37
35
  self.chunk_overlap = chunk_overlap
38
36
 
@@ -50,13 +48,8 @@ class TextChunker(ABC):
50
48
  ) -> list[TextChunk]:
51
49
  """Split multiple texts into chunks."""
52
50
  result: list[TextChunk] = []
53
-
54
51
  for item in texts:
55
- if isinstance(item, tuple):
56
- text, metadata = item
57
- else:
58
- text, metadata = item, None
52
+ text, metadata = item if isinstance(item, tuple) else (item, None)
59
53
  chunks = await self.split(text, metadata)
60
54
  result.extend(chunks)
61
-
62
55
  return result
@@ -0,0 +1,5 @@
1
+ """LlamaIndex Chunker."""
2
+
3
+ from docler.chunkers.llamaindex_chunker.chunker import LlamaIndexChunker
4
+
5
+ __all__ = ["LlamaIndexChunker"]
@@ -0,0 +1,167 @@
1
+ """LlamaIndex-based text chunking implementation."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import logging
6
+ from typing import TYPE_CHECKING, Any, ClassVar, Literal
7
+
8
+ from docler.chunkers.base import TextChunk, TextChunker
9
+
10
+
11
+ if TYPE_CHECKING:
12
+ from docler.models import Document
13
+
14
+
15
+ logger = logging.getLogger(__name__)
16
+
17
+ ChunkerType = Literal["sentence", "token", "fixed", "markdown"]
18
+
19
+
20
+ class LlamaIndexChunker(TextChunker):
21
+ """Text chunker using LlamaIndex chunkers.
22
+
23
+ This is a wrapper around LlamaIndex's chunking functionality,
24
+ allowing dynamic import to avoid a fixed dependency.
25
+ """
26
+
27
+ REQUIRED_PACKAGES: ClassVar = {"llama-index"}
28
+
29
+ def __init__(
30
+ self,
31
+ *,
32
+ chunker_type: ChunkerType = "markdown",
33
+ chunk_size: int = 1000,
34
+ chunk_overlap: int = 200,
35
+ include_metadata: bool = True,
36
+ include_prev_next_rel: bool = False,
37
+ ):
38
+ """Initialize the LlamaIndex chunker.
39
+
40
+ Args:
41
+ chunker_type: Which LlamaIndex chunker to use
42
+ chunk_size: Target size of chunks in appropriate units
43
+ chunk_overlap: Overlap between chunks
44
+ include_metadata: Whether to include document metadata in chunks
45
+ include_prev_next_rel: Whether to track relationships between chunks
46
+ """
47
+ super().__init__(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
48
+ self.chunker_type = chunker_type
49
+ self.include_metadata = include_metadata
50
+ self.include_prev_next_rel = include_prev_next_rel
51
+
52
+ def _get_llama_chunker(self):
53
+ """Dynamically import and create a LlamaIndex chunker.
54
+
55
+ Returns:
56
+ The appropriate LlamaIndex chunker instance based on configuration
57
+
58
+ Raises:
59
+ ImportError: If llama-index isn't installed
60
+ """
61
+ if self.chunker_type == "sentence":
62
+ from llama_index.core.node_parser import SentenceSplitter
63
+
64
+ return SentenceSplitter(
65
+ chunk_size=self.chunk_size,
66
+ chunk_overlap=self.chunk_overlap,
67
+ include_metadata=self.include_metadata,
68
+ include_prev_next_rel=self.include_prev_next_rel,
69
+ )
70
+ if self.chunker_type == "token":
71
+ from llama_index.core.node_parser import TokenTextSplitter
72
+
73
+ return TokenTextSplitter(
74
+ chunk_size=self.chunk_size,
75
+ chunk_overlap=self.chunk_overlap,
76
+ include_metadata=self.include_metadata,
77
+ include_prev_next_rel=self.include_prev_next_rel,
78
+ )
79
+ if self.chunker_type == "fixed":
80
+ from llama_index.core.node_parser import SentenceWindowNodeParser
81
+
82
+ return SentenceWindowNodeParser.from_defaults(
83
+ window_size=self.chunk_size,
84
+ window_metadata_key="window",
85
+ original_text_metadata_key="original_text",
86
+ )
87
+ # markdown as default
88
+ from llama_index.core.node_parser import MarkdownNodeParser
89
+
90
+ return MarkdownNodeParser(
91
+ include_metadata=self.include_metadata,
92
+ include_prev_next_rel=self.include_prev_next_rel,
93
+ )
94
+
95
+ def _find_images_for_chunk(self, doc: Document, chunk_text: str) -> list:
96
+ """Find images that are referenced in the chunk.
97
+
98
+ Args:
99
+ doc: Original document
100
+ chunk_text: Text of the current chunk
101
+
102
+ Returns:
103
+ List of images referenced in the chunk
104
+ """
105
+ return [img for img in doc.images if img.filename and img.filename in chunk_text]
106
+
107
+ async def split(
108
+ self,
109
+ doc: Document,
110
+ extra_metadata: dict[str, Any] | None = None,
111
+ ) -> list[TextChunk]:
112
+ """Split document into chunks using LlamaIndex.
113
+
114
+ Args:
115
+ doc: Document to split
116
+ extra_metadata: Additional metadata to include in chunks
117
+
118
+ Returns:
119
+ List of text chunks
120
+
121
+ Raises:
122
+ ImportError: If llama-index isn't installed
123
+ """
124
+ from llama_index import core
125
+
126
+ meta = {
127
+ "source": doc.source_path or "",
128
+ "title": doc.title or "",
129
+ **(extra_metadata or {}),
130
+ }
131
+ llama_doc = core.Document(text=doc.content, metadata=meta)
132
+ chunker = self._get_llama_chunker()
133
+ nodes = chunker.get_nodes_from_documents([llama_doc])
134
+ chunks = []
135
+ for i, node in enumerate(nodes):
136
+ metadata = {**node.metadata}
137
+ metadata["relationships"] = node.relationships
138
+ chunk_images = self._find_images_for_chunk(doc, node.get_content())
139
+ chunk = TextChunk(
140
+ text=node.get_content(),
141
+ source_doc_id=doc.source_path or "",
142
+ chunk_index=i,
143
+ images=chunk_images,
144
+ metadata={**(extra_metadata or {}), **metadata},
145
+ )
146
+ chunks.append(chunk)
147
+
148
+ return chunks
149
+
150
+
151
+ if __name__ == "__main__":
152
+ import asyncio
153
+
154
+ from docler.models import Document
155
+
156
+ async def main():
157
+ # Example usage
158
+ doc = Document(
159
+ source_path="example.txt",
160
+ content="# This is an example\n\nSample content here.",
161
+ )
162
+ chunker = LlamaIndexChunker(chunker_type="markdown")
163
+ chunks = await chunker.split(doc)
164
+ for chunk in chunks:
165
+ print(f"Chunk {chunk.chunk_index}:\n{chunk.text}\n")
166
+
167
+ asyncio.run(main())
@@ -0,0 +1,5 @@
1
+ """Markdown Chunker."""
2
+
3
+ from docler.chunkers.markdown_chunker.chunker import MarkdownChunker
4
+
5
+ __all__ = ["MarkdownChunker"]