docler 0.1.2__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {docler-0.1.2 → docler-0.2.0}/.pre-commit-config.yaml +1 -1
- {docler-0.1.2 → docler-0.2.0}/PKG-INFO +22 -6
- {docler-0.1.2 → docler-0.2.0}/pyproject.toml +22 -4
- docler-0.2.0/src/docler/__init__.py +34 -0
- docler-0.2.0/src/docler/__main__.py +55 -0
- docler-0.2.0/src/docler/chunkers/ai_chunker/__init__.py +5 -0
- docler-0.2.0/src/docler/chunkers/ai_chunker/chunker.py +161 -0
- {docler-0.1.2 → docler-0.2.0}/src/docler/chunkers/base.py +5 -12
- docler-0.2.0/src/docler/chunkers/llamaindex_chunker/__init__.py +5 -0
- docler-0.2.0/src/docler/chunkers/llamaindex_chunker/chunker.py +167 -0
- docler-0.2.0/src/docler/chunkers/markdown_chunker/__init__.py +5 -0
- docler-0.2.0/src/docler/chunkers/markdown_chunker/chunker.py +152 -0
- docler-0.2.0/src/docler/common_types.py +37 -0
- docler-0.2.0/src/docler/configs/chunker_configs.py +84 -0
- docler-0.1.2/src/docler/configs.py → docler-0.2.0/src/docler/configs/converter_configs.py +21 -16
- docler-0.2.0/src/docler/configs/vector_db_configs.py +100 -0
- docler-0.2.0/src/docler/converters/__init__.py +0 -0
- docler-0.2.0/src/docler/converters/azure_provider/__init__.py +0 -0
- docler-0.2.0/src/docler/converters/azure_provider/converter.py +237 -0
- {docler-0.1.2/src/docler → docler-0.2.0/src/docler/converters}/base.py +6 -6
- docler-0.2.0/src/docler/converters/datalab_provider/__init__.py +5 -0
- {docler-0.1.2/src/docler → docler-0.2.0/src/docler/converters}/datalab_provider/provider.py +14 -30
- {docler-0.1.2/src/docler → docler-0.2.0/src/docler/converters}/dir_converter.py +3 -8
- docler-0.2.0/src/docler/converters/docling_provider/__init__.py +5 -0
- {docler-0.1.2/src/docler → docler-0.2.0/src/docler/converters}/docling_provider/provider.py +56 -56
- docler-0.1.2/src/docler/lang_code.py → docler-0.2.0/src/docler/converters/docling_provider/utils.py +17 -34
- docler-0.2.0/src/docler/converters/kreuzberg_provider/__init__.py +5 -0
- {docler-0.1.2/src/docler → docler-0.2.0/src/docler/converters}/kreuzberg_provider/provider.py +6 -8
- docler-0.2.0/src/docler/converters/llamaindex_provider/__init__.py +5 -0
- docler-0.2.0/src/docler/converters/llamaindex_provider/provider.py +124 -0
- docler-0.2.0/src/docler/converters/llamaparse_provider/__init__.py +5 -0
- docler-0.2.0/src/docler/converters/llamaparse_provider/provider.py +151 -0
- docler-0.2.0/src/docler/converters/llm_provider/__init__.py +5 -0
- {docler-0.1.2/src/docler/litellm_provider → docler-0.2.0/src/docler/converters/llm_provider}/provider.py +27 -51
- docler-0.2.0/src/docler/converters/marker_provider/__init__.py +5 -0
- {docler-0.1.2/src/docler → docler-0.2.0/src/docler/converters}/marker_provider/provider.py +18 -23
- docler-0.2.0/src/docler/converters/markitdown_provider/__init__.py +5 -0
- {docler-0.1.2/src/docler → docler-0.2.0/src/docler/converters}/markitdown_provider/provider.py +5 -12
- docler-0.2.0/src/docler/converters/mistral_provider/__init__.py +5 -0
- {docler-0.1.2/src/docler → docler-0.2.0/src/docler/converters}/mistral_provider/provider.py +10 -27
- {docler-0.1.2/src/docler → docler-0.2.0/src/docler/converters}/registry.py +6 -7
- docler-0.2.0/src/docler/embeddings/__init__.py +1 -0
- docler-0.2.0/src/docler/embeddings/base.py +63 -0
- docler-0.2.0/src/docler/embeddings/bge_provider.py +47 -0
- docler-0.2.0/src/docler/embeddings/litellm_provider.py +109 -0
- docler-0.2.0/src/docler/embeddings/openai_provider.py +106 -0
- docler-0.2.0/src/docler/embeddings/stf_provider.py +51 -0
- docler-0.2.0/src/docler/provider.py +25 -0
- docler-0.2.0/src/docler/py.typed +0 -0
- docler-0.2.0/src/docler/resources/pdf_sample.pdf +0 -0
- docler-0.2.0/src/docler/streamlit_app/__init__.py +5 -0
- docler-0.2.0/src/docler/streamlit_app/app.py +58 -0
- docler-0.2.0/src/docler/streamlit_app/chunkers.py +15 -0
- docler-0.2.0/src/docler/streamlit_app/converters.py +23 -0
- docler-0.2.0/src/docler/streamlit_app/state.py +53 -0
- docler-0.2.0/src/docler/streamlit_app/step1_conversion.py +108 -0
- docler-0.2.0/src/docler/streamlit_app/step2_chunking.py +201 -0
- docler-0.2.0/src/docler/streamlit_app/utils.py +36 -0
- docler-0.2.0/src/docler/streamlit_app.py +145 -0
- docler-0.2.0/src/docler/streamlit_chunk_app.py +127 -0
- docler-0.2.0/src/docler/utils.py +74 -0
- docler-0.2.0/src/docler/vector_db/__init__.py +1 -0
- docler-0.2.0/src/docler/vector_db/base.py +447 -0
- docler-0.2.0/src/docler/vector_db/dbs/__init__.py +0 -0
- docler-0.2.0/src/docler/vector_db/dbs/chroma_db/__init__.py +5 -0
- docler-0.2.0/src/docler/vector_db/dbs/chroma_db/db.py +259 -0
- docler-0.2.0/src/docler/vector_db/dbs/openai_db/__init__.py +0 -0
- docler-0.2.0/src/docler/vector_db/dbs/openai_db/db.py +327 -0
- docler-0.2.0/src/docler/vector_db/dbs/openai_db/manager.py +301 -0
- docler-0.2.0/src/docler/vector_db/dbs/qdrant_db/__init__.py +5 -0
- docler-0.2.0/src/docler/vector_db/dbs/qdrant_db/db.py +334 -0
- docler-0.2.0/src/docler/vector_db/dbs/upstash_db/__init__.py +7 -0
- docler-0.2.0/src/docler/vector_db/dbs/upstash_db/db.py +426 -0
- docler-0.2.0/src/docler/vector_db/dbs/upstash_db/manager.py +298 -0
- docler-0.2.0/tests/resources/pdf_sample.pdf +0 -0
- docler-0.2.0/tests/test_chunkers.py +156 -0
- docler-0.1.2/src/docler/__init__.py +0 -34
- docler-0.1.2/src/docler/__main__.py +0 -81
- docler-0.1.2/src/docler/common_types.py +0 -6
- docler-0.1.2/src/docler/datalab_provider/__init__.py +0 -5
- docler-0.1.2/src/docler/docling_provider/__init__.py +0 -5
- docler-0.1.2/src/docler/kreuzberg_provider/__init__.py +0 -5
- docler-0.1.2/src/docler/litellm_provider/__init__.py +0 -5
- docler-0.1.2/src/docler/marker_provider/__init__.py +0 -5
- docler-0.1.2/src/docler/markitdown_provider/__init__.py +0 -5
- docler-0.1.2/src/docler/mistral_provider/__init__.py +0 -5
- docler-0.1.2/src/docler/olmocr_provider/__init__.py +0 -5
- docler-0.1.2/src/docler/olmocr_provider/provider.py +0 -196
- docler-0.1.2/src/docler/streamlit_app.py +0 -103
- docler-0.1.2/src/docler/utils.py +0 -21
- {docler-0.1.2 → docler-0.2.0}/.copier-answers.yml +0 -0
- {docler-0.1.2 → docler-0.2.0}/.github/FUNDING.yml +0 -0
- {docler-0.1.2 → docler-0.2.0}/.github/copilot-instructions.md +0 -0
- {docler-0.1.2 → docler-0.2.0}/.github/dependabot.yml +0 -0
- {docler-0.1.2 → docler-0.2.0}/.github/workflows/build.yml +0 -0
- {docler-0.1.2 → docler-0.2.0}/.github/workflows/documentation.yml +0 -0
- {docler-0.1.2 → docler-0.2.0}/.gitignore +0 -0
- {docler-0.1.2 → docler-0.2.0}/LICENSE +0 -0
- {docler-0.1.2 → docler-0.2.0}/README.md +0 -0
- {docler-0.1.2 → docler-0.2.0}/docs/.empty +0 -0
- {docler-0.1.2 → docler-0.2.0}/duties.py +0 -0
- {docler-0.1.2 → docler-0.2.0}/mkdocs.yml +0 -0
- {docler-0.1.2 → docler-0.2.0}/overrides/_dummy.txt +0 -0
- {docler-0.1.2 → docler-0.2.0}/src/docler/chunkers/__init__.py +0 -0
- /docler-0.1.2/src/docler/py.typed → /docler-0.2.0/src/docler/configs/__init__.py +0 -0
- {docler-0.1.2 → docler-0.2.0}/src/docler/models.py +0 -0
- {docler-0.1.2 → docler-0.2.0}/tests/__init__.py +0 -0
- {docler-0.1.2 → docler-0.2.0}/tests/conftest.py +0 -0
- {docler-0.1.2 → docler-0.2.0}/tests/test_converters.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: docler
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: Abstractions & Tools for OCR / document processing
|
|
5
5
|
Project-URL: Documentation, https://phil65.github.io/docler/
|
|
6
6
|
Project-URL: Source, https://github.com/phil65/docler
|
|
@@ -48,13 +48,20 @@ Requires-Dist: anyenv>=0.4.1
|
|
|
48
48
|
Requires-Dist: pydantic
|
|
49
49
|
Requires-Dist: upathtools>=0.4.3
|
|
50
50
|
Provides-Extra: all
|
|
51
|
+
Requires-Dist: chromadb; extra == 'all'
|
|
51
52
|
Requires-Dist: docling; extra == 'all'
|
|
52
53
|
Requires-Dist: kreuzberg; extra == 'all'
|
|
53
|
-
Requires-Dist:
|
|
54
|
-
Requires-Dist:
|
|
55
|
-
Requires-Dist:
|
|
54
|
+
Requires-Dist: llama-index-readers-smart-pdf-loader; extra == 'all'
|
|
55
|
+
Requires-Dist: llmling-agent[pydantic-ai]; extra == 'all'
|
|
56
|
+
Requires-Dist: llmsherpa; extra == 'all'
|
|
57
|
+
Requires-Dist: marker-pdf; extra == 'all'
|
|
58
|
+
Requires-Dist: markitdown; extra == 'all'
|
|
56
59
|
Requires-Dist: mistralai; extra == 'all'
|
|
60
|
+
Requires-Dist: openai; extra == 'all'
|
|
57
61
|
Requires-Dist: streamlit; extra == 'all'
|
|
62
|
+
Requires-Dist: upstash-vector; extra == 'all'
|
|
63
|
+
Provides-Extra: chromadb
|
|
64
|
+
Requires-Dist: chromadb>=0.6.3; extra == 'chromadb'
|
|
58
65
|
Provides-Extra: docling
|
|
59
66
|
Requires-Dist: docling; extra == 'docling'
|
|
60
67
|
Provides-Extra: kreuzberg
|
|
@@ -62,14 +69,23 @@ Requires-Dist: kreuzberg; extra == 'kreuzberg'
|
|
|
62
69
|
Provides-Extra: litellm
|
|
63
70
|
Requires-Dist: litellm; extra == 'litellm'
|
|
64
71
|
Requires-Dist: tokonomics; extra == 'litellm'
|
|
72
|
+
Provides-Extra: llama-index
|
|
73
|
+
Requires-Dist: llama-index; extra == 'llama-index'
|
|
74
|
+
Provides-Extra: llama-parse
|
|
75
|
+
Requires-Dist: llama-parse; extra == 'llama-parse'
|
|
65
76
|
Provides-Extra: marker
|
|
66
77
|
Requires-Dist: marker-pdf; extra == 'marker'
|
|
67
|
-
Provides-Extra: markitdown
|
|
68
|
-
Requires-Dist: markitdown[all]>=0.1.0a1; extra == 'markitdown'
|
|
69
78
|
Provides-Extra: mistralai
|
|
70
79
|
Requires-Dist: mistralai; extra == 'mistralai'
|
|
80
|
+
Provides-Extra: openai
|
|
81
|
+
Requires-Dist: openai; extra == 'openai'
|
|
82
|
+
Provides-Extra: smart-pdf
|
|
83
|
+
Requires-Dist: llama-index-readers-smart-pdf-loader; extra == 'smart-pdf'
|
|
84
|
+
Requires-Dist: llmsherpa; extra == 'smart-pdf'
|
|
71
85
|
Provides-Extra: streamlit
|
|
72
86
|
Requires-Dist: streamlit; extra == 'streamlit'
|
|
87
|
+
Provides-Extra: upstash
|
|
88
|
+
Requires-Dist: upstash-vector; extra == 'upstash'
|
|
73
89
|
Description-Content-Type: text/markdown
|
|
74
90
|
|
|
75
91
|
# Docler
|
|
@@ -45,19 +45,34 @@ marker = ["marker-pdf"]
|
|
|
45
45
|
mistralai = ["mistralai"]
|
|
46
46
|
docling = ["docling"]
|
|
47
47
|
# olmocr = ["olmocr", "accelerate"]
|
|
48
|
-
markitdown = ["markitdown[all]
|
|
48
|
+
# markitdown = ["markitdown[all]"]
|
|
49
49
|
kreuzberg = ["kreuzberg"]
|
|
50
50
|
litellm = ["litellm", "tokonomics"]
|
|
51
51
|
streamlit = ["streamlit"]
|
|
52
|
+
llama-index = ["llama-index"]
|
|
53
|
+
llama-parse = ["llama-parse"]
|
|
54
|
+
smart-pdf = ["llmsherpa", "llama-index-readers-smart-pdf-loader"]
|
|
55
|
+
chromadb = ["chromadb>=0.6.3"]
|
|
56
|
+
upstash = ["upstash-vector"]
|
|
57
|
+
openai = ["openai"]
|
|
52
58
|
all = [
|
|
53
|
-
"marker",
|
|
59
|
+
"marker-pdf",
|
|
54
60
|
"mistralai",
|
|
55
61
|
"docling",
|
|
56
|
-
"markitdown
|
|
62
|
+
"markitdown",
|
|
57
63
|
"kreuzberg",
|
|
58
|
-
"litellm",
|
|
64
|
+
# "litellm",
|
|
65
|
+
"llmling-agent[pydantic-ai]",
|
|
66
|
+
# "llama-parse",
|
|
59
67
|
"streamlit",
|
|
68
|
+
"llmsherpa",
|
|
69
|
+
"llama-index-readers-smart-pdf-loader",
|
|
70
|
+
"chromadb",
|
|
71
|
+
"upstash-vector",
|
|
72
|
+
"openai",
|
|
60
73
|
]
|
|
74
|
+
|
|
75
|
+
|
|
61
76
|
[project.scripts]
|
|
62
77
|
docler = "docler.__main__:cli"
|
|
63
78
|
|
|
@@ -70,6 +85,7 @@ dev = [
|
|
|
70
85
|
"pytest-cov",
|
|
71
86
|
"pyreadline3",
|
|
72
87
|
"devtools",
|
|
88
|
+
"pytest-asyncio",
|
|
73
89
|
# Only add below (Copier)
|
|
74
90
|
]
|
|
75
91
|
benchmark = ["pyinstrument"]
|
|
@@ -97,6 +113,7 @@ testpaths = "tests/"
|
|
|
97
113
|
log_cli = true
|
|
98
114
|
log_format = "%(asctime)s %(levelname)s %(message)s"
|
|
99
115
|
log_date_format = "%Y-%m-%d %H:%M:%S"
|
|
116
|
+
asyncio_mode = "auto"
|
|
100
117
|
|
|
101
118
|
[tool.coverage.report]
|
|
102
119
|
exclude_lines = [
|
|
@@ -215,6 +232,7 @@ ignore = [
|
|
|
215
232
|
# "PLR2004", # Magic values instead of named consts
|
|
216
233
|
"SLF001", # Private member accessed
|
|
217
234
|
"TRY003", # Avoid specifying long messages outside the exception class
|
|
235
|
+
"TC006",
|
|
218
236
|
]
|
|
219
237
|
|
|
220
238
|
[tool.ruff.lint.flake8-quotes]
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""Document conversion library supporting multiple providers."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from docler.converters.base import DocumentConverter
|
|
6
|
+
from docler.converters.dir_converter import Conversion, DirectoryConverter
|
|
7
|
+
from docler.models import Document, Image, ImageReferenceFormat
|
|
8
|
+
from docler.converters.registry import ConverterRegistry
|
|
9
|
+
|
|
10
|
+
# Import providers
|
|
11
|
+
from docler.converters.docling_provider import DoclingConverter
|
|
12
|
+
from docler.converters.marker_provider import MarkerConverter
|
|
13
|
+
from docler.converters.mistral_provider import MistralConverter
|
|
14
|
+
from docler.converters.llm_provider import LLMConverter
|
|
15
|
+
from docler.converters.datalab_provider import DataLabConverter
|
|
16
|
+
from docler.converters.llamaparse_provider import LlamaParseConverter
|
|
17
|
+
|
|
18
|
+
__version__ = "0.2.0"
|
|
19
|
+
|
|
20
|
+
__all__ = [
|
|
21
|
+
"Conversion",
|
|
22
|
+
"ConverterRegistry",
|
|
23
|
+
"DataLabConverter",
|
|
24
|
+
"DirectoryConverter",
|
|
25
|
+
"DoclingConverter",
|
|
26
|
+
"Document",
|
|
27
|
+
"DocumentConverter",
|
|
28
|
+
"Image",
|
|
29
|
+
"ImageReferenceFormat",
|
|
30
|
+
"LLMConverter",
|
|
31
|
+
"LlamaParseConverter",
|
|
32
|
+
"MarkerConverter",
|
|
33
|
+
"MistralConverter",
|
|
34
|
+
]
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
"""Command line interface for Docler document converter."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
import os
|
|
7
|
+
import pathlib
|
|
8
|
+
import subprocess
|
|
9
|
+
import sys
|
|
10
|
+
|
|
11
|
+
import typer
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
cli = typer.Typer(help="Docler document converter CLI", no_args_is_help=True)
|
|
15
|
+
|
|
16
|
+
logger = logging.getLogger(__name__)
|
|
17
|
+
|
|
18
|
+
PACKAGE_DIR = pathlib.Path(__file__).parent
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@cli.command()
|
|
22
|
+
def serve():
|
|
23
|
+
"""Start the Streamlit web interface."""
|
|
24
|
+
app_path = str(PACKAGE_DIR / "streamlit_app" / "app.py")
|
|
25
|
+
try:
|
|
26
|
+
cmd = [sys.executable, "-m", "streamlit", "run", app_path]
|
|
27
|
+
subprocess.run(cmd, env=os.environ.copy(), check=True)
|
|
28
|
+
except subprocess.CalledProcessError as e:
|
|
29
|
+
# msg = f"Failed to start Streamlit: {e}"
|
|
30
|
+
raise typer.Exit(1) from e
|
|
31
|
+
except KeyboardInterrupt:
|
|
32
|
+
logger.info("Shutting down...")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@cli.command("chunk_ui")
|
|
36
|
+
def chunk_ui():
|
|
37
|
+
"""Start the Streamlit web interface."""
|
|
38
|
+
app_path = str(PACKAGE_DIR / "streamlit_chunk_app.py")
|
|
39
|
+
try:
|
|
40
|
+
cmd = [sys.executable, "-m", "streamlit", "run", app_path]
|
|
41
|
+
subprocess.run(cmd, env=os.environ.copy(), check=True)
|
|
42
|
+
except subprocess.CalledProcessError as e:
|
|
43
|
+
# msg = f"Failed to start Streamlit: {e}"
|
|
44
|
+
raise typer.Exit(1) from e
|
|
45
|
+
except KeyboardInterrupt:
|
|
46
|
+
logger.info("Shutting down...")
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def main():
|
|
50
|
+
"""Main entry point."""
|
|
51
|
+
cli()
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
if __name__ == "__main__":
|
|
55
|
+
main()
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
"""AI-based markdown chunking implementation."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import TYPE_CHECKING, Any, ClassVar
|
|
6
|
+
|
|
7
|
+
from pydantic import BaseModel, ConfigDict
|
|
8
|
+
|
|
9
|
+
from docler.chunkers.base import TextChunk, TextChunker
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
if TYPE_CHECKING:
|
|
13
|
+
from llmling_agent.agent.agent import AgentType
|
|
14
|
+
|
|
15
|
+
from docler.models import Document
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class Chunk(BaseModel):
|
|
19
|
+
"""A chunk of text with semantic metadata."""
|
|
20
|
+
|
|
21
|
+
start_row: int
|
|
22
|
+
"""Start line number (1-based)"""
|
|
23
|
+
|
|
24
|
+
end_row: int
|
|
25
|
+
"""End line number (1-based)"""
|
|
26
|
+
|
|
27
|
+
keywords: list[str]
|
|
28
|
+
"""Key terms and concepts in this chunk"""
|
|
29
|
+
|
|
30
|
+
references: list[int]
|
|
31
|
+
"""Line numbers that this chunk references or depends on"""
|
|
32
|
+
|
|
33
|
+
model_config = ConfigDict(use_attribute_docstrings=True)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class Chunks(BaseModel):
|
|
37
|
+
"""Collection of chunks with their metadata."""
|
|
38
|
+
|
|
39
|
+
chunks: list[Chunk]
|
|
40
|
+
"""A list of chunks to extract from the document."""
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
SYS_PROMPT = """
|
|
44
|
+
You are an expert at dividing text into meaningful chunks while preserving context and relationships.
|
|
45
|
+
|
|
46
|
+
Analyze this text and split it into coherent chunks. For each chunk:
|
|
47
|
+
1. Define its start and end line numbers (1-based)
|
|
48
|
+
2. Extract key terms and concepts as keywords
|
|
49
|
+
3. Note any line numbers it references or depends on
|
|
50
|
+
|
|
51
|
+
Rules:
|
|
52
|
+
- Each chunk should be semantically complete
|
|
53
|
+
- Identify cross-references between different parts of the text
|
|
54
|
+
- Keywords should be specific and relevant
|
|
55
|
+
- Line numbers must be accurate
|
|
56
|
+
|
|
57
|
+
""" # noqa: E501
|
|
58
|
+
|
|
59
|
+
CHUNKING_PROMPT = """
|
|
60
|
+
Here's the text with line numbers:
|
|
61
|
+
|
|
62
|
+
{numbered_text}
|
|
63
|
+
|
|
64
|
+
"""
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class AIChunker(TextChunker):
|
|
68
|
+
"""LLM-based document chunker."""
|
|
69
|
+
|
|
70
|
+
REQUIRED_PACKAGES: ClassVar = {"llmling-agent"}
|
|
71
|
+
|
|
72
|
+
def __init__(
|
|
73
|
+
self,
|
|
74
|
+
model: str = "openrouter:openai/o3-mini", # google/gemini-2.0-flash-lite-001
|
|
75
|
+
provider: AgentType = "pydantic_ai",
|
|
76
|
+
min_chunk_size: int = 200,
|
|
77
|
+
max_chunk_size: int = 1500,
|
|
78
|
+
):
|
|
79
|
+
"""Initialize the AI chunker.
|
|
80
|
+
|
|
81
|
+
Args:
|
|
82
|
+
model: LLM model to use
|
|
83
|
+
provider: LLM provider to use
|
|
84
|
+
min_chunk_size: Minimum characters per chunk
|
|
85
|
+
max_chunk_size: Maximum characters per chunk
|
|
86
|
+
"""
|
|
87
|
+
self.model = model
|
|
88
|
+
self.provider = provider
|
|
89
|
+
self.min_chunk_size = min_chunk_size
|
|
90
|
+
self.max_chunk_size = max_chunk_size
|
|
91
|
+
|
|
92
|
+
def _add_line_numbers(self, text: str) -> str:
|
|
93
|
+
"""Add line numbers to text."""
|
|
94
|
+
lines = text.splitlines()
|
|
95
|
+
return "\n".join(f"{i + 1:3d} | {line}" for i, line in enumerate(lines))
|
|
96
|
+
|
|
97
|
+
async def _get_chunks(self, text: str) -> Chunks:
|
|
98
|
+
"""Get chunk definitions from LLM."""
|
|
99
|
+
import llmling_agent
|
|
100
|
+
|
|
101
|
+
numbered_text = self._add_line_numbers(text)
|
|
102
|
+
agent: llmling_agent.StructuredAgent[None, Chunks] = llmling_agent.Agent(
|
|
103
|
+
model=self.model,
|
|
104
|
+
provider=self.provider, # pyright: ignore
|
|
105
|
+
system_prompt=SYS_PROMPT,
|
|
106
|
+
).to_structured(Chunks)
|
|
107
|
+
prompt = CHUNKING_PROMPT.format(numbered_text=numbered_text)
|
|
108
|
+
response = await agent.run(prompt)
|
|
109
|
+
return response.content
|
|
110
|
+
|
|
111
|
+
def _create_text_chunk(
|
|
112
|
+
self,
|
|
113
|
+
doc: Document,
|
|
114
|
+
chunk: Chunk,
|
|
115
|
+
chunk_idx: int,
|
|
116
|
+
extra_metadata: dict[str, Any] | None = None,
|
|
117
|
+
) -> TextChunk:
|
|
118
|
+
"""Create a TextChunk from chunk definition."""
|
|
119
|
+
lines = doc.content.splitlines()
|
|
120
|
+
chunk_lines = lines[chunk.start_row - 1 : chunk.end_row]
|
|
121
|
+
chunk_text = "\n".join(chunk_lines)
|
|
122
|
+
base = extra_metadata or {}
|
|
123
|
+
metadata = {**base, "keywords": chunk.keywords, "references": chunk.references}
|
|
124
|
+
chunk_images = [i for i in doc.images if i.filename and i.filename in chunk_text]
|
|
125
|
+
return TextChunk(
|
|
126
|
+
text=chunk_text,
|
|
127
|
+
source_doc_id=doc.source_path or "",
|
|
128
|
+
chunk_index=chunk_idx,
|
|
129
|
+
images=chunk_images,
|
|
130
|
+
metadata=metadata,
|
|
131
|
+
)
|
|
132
|
+
|
|
133
|
+
async def split(
|
|
134
|
+
self,
|
|
135
|
+
doc: Document,
|
|
136
|
+
extra_metadata: dict[str, Any] | None = None,
|
|
137
|
+
) -> list[TextChunk]:
|
|
138
|
+
"""Split document into chunks using LLM analysis."""
|
|
139
|
+
# Get chunk definitions from LLM
|
|
140
|
+
chunks = await self._get_chunks(doc.content)
|
|
141
|
+
|
|
142
|
+
# Convert to TextChunks
|
|
143
|
+
return [
|
|
144
|
+
self._create_text_chunk(doc, chunk, i, extra_metadata)
|
|
145
|
+
for i, chunk in enumerate(chunks.chunks)
|
|
146
|
+
]
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
if __name__ == "__main__":
|
|
150
|
+
import asyncio
|
|
151
|
+
|
|
152
|
+
from docler.models import Document
|
|
153
|
+
|
|
154
|
+
async def main():
|
|
155
|
+
# Example usage
|
|
156
|
+
doc = Document(source_path="example.txt", content=SYS_PROMPT)
|
|
157
|
+
chunker = AIChunker()
|
|
158
|
+
chunks = await chunker.split(doc)
|
|
159
|
+
print(chunks)
|
|
160
|
+
|
|
161
|
+
asyncio.run(main())
|
|
@@ -6,6 +6,8 @@ from abc import ABC, abstractmethod
|
|
|
6
6
|
from dataclasses import dataclass, field
|
|
7
7
|
from typing import TYPE_CHECKING, Any
|
|
8
8
|
|
|
9
|
+
from docler.provider import BaseProvider
|
|
10
|
+
|
|
9
11
|
|
|
10
12
|
if TYPE_CHECKING:
|
|
11
13
|
from collections.abc import Iterable
|
|
@@ -25,14 +27,10 @@ class TextChunk:
|
|
|
25
27
|
metadata: dict[str, Any] = field(default_factory=dict)
|
|
26
28
|
|
|
27
29
|
|
|
28
|
-
class TextChunker(ABC):
|
|
30
|
+
class TextChunker(BaseProvider, ABC):
|
|
29
31
|
"""Base class for text chunkers."""
|
|
30
32
|
|
|
31
|
-
def __init__(
|
|
32
|
-
self,
|
|
33
|
-
chunk_size: int = 500,
|
|
34
|
-
chunk_overlap: int = 50,
|
|
35
|
-
) -> None:
|
|
33
|
+
def __init__(self, chunk_size: int = 500, chunk_overlap: int = 50):
|
|
36
34
|
self.chunk_size = chunk_size
|
|
37
35
|
self.chunk_overlap = chunk_overlap
|
|
38
36
|
|
|
@@ -50,13 +48,8 @@ class TextChunker(ABC):
|
|
|
50
48
|
) -> list[TextChunk]:
|
|
51
49
|
"""Split multiple texts into chunks."""
|
|
52
50
|
result: list[TextChunk] = []
|
|
53
|
-
|
|
54
51
|
for item in texts:
|
|
55
|
-
if isinstance(item, tuple)
|
|
56
|
-
text, metadata = item
|
|
57
|
-
else:
|
|
58
|
-
text, metadata = item, None
|
|
52
|
+
text, metadata = item if isinstance(item, tuple) else (item, None)
|
|
59
53
|
chunks = await self.split(text, metadata)
|
|
60
54
|
result.extend(chunks)
|
|
61
|
-
|
|
62
55
|
return result
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
"""LlamaIndex-based text chunking implementation."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
from typing import TYPE_CHECKING, Any, ClassVar, Literal
|
|
7
|
+
|
|
8
|
+
from docler.chunkers.base import TextChunk, TextChunker
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
if TYPE_CHECKING:
|
|
12
|
+
from docler.models import Document
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
logger = logging.getLogger(__name__)
|
|
16
|
+
|
|
17
|
+
ChunkerType = Literal["sentence", "token", "fixed", "markdown"]
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class LlamaIndexChunker(TextChunker):
|
|
21
|
+
"""Text chunker using LlamaIndex chunkers.
|
|
22
|
+
|
|
23
|
+
This is a wrapper around LlamaIndex's chunking functionality,
|
|
24
|
+
allowing dynamic import to avoid a fixed dependency.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
REQUIRED_PACKAGES: ClassVar = {"llama-index"}
|
|
28
|
+
|
|
29
|
+
def __init__(
|
|
30
|
+
self,
|
|
31
|
+
*,
|
|
32
|
+
chunker_type: ChunkerType = "markdown",
|
|
33
|
+
chunk_size: int = 1000,
|
|
34
|
+
chunk_overlap: int = 200,
|
|
35
|
+
include_metadata: bool = True,
|
|
36
|
+
include_prev_next_rel: bool = False,
|
|
37
|
+
):
|
|
38
|
+
"""Initialize the LlamaIndex chunker.
|
|
39
|
+
|
|
40
|
+
Args:
|
|
41
|
+
chunker_type: Which LlamaIndex chunker to use
|
|
42
|
+
chunk_size: Target size of chunks in appropriate units
|
|
43
|
+
chunk_overlap: Overlap between chunks
|
|
44
|
+
include_metadata: Whether to include document metadata in chunks
|
|
45
|
+
include_prev_next_rel: Whether to track relationships between chunks
|
|
46
|
+
"""
|
|
47
|
+
super().__init__(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
|
|
48
|
+
self.chunker_type = chunker_type
|
|
49
|
+
self.include_metadata = include_metadata
|
|
50
|
+
self.include_prev_next_rel = include_prev_next_rel
|
|
51
|
+
|
|
52
|
+
def _get_llama_chunker(self):
|
|
53
|
+
"""Dynamically import and create a LlamaIndex chunker.
|
|
54
|
+
|
|
55
|
+
Returns:
|
|
56
|
+
The appropriate LlamaIndex chunker instance based on configuration
|
|
57
|
+
|
|
58
|
+
Raises:
|
|
59
|
+
ImportError: If llama-index isn't installed
|
|
60
|
+
"""
|
|
61
|
+
if self.chunker_type == "sentence":
|
|
62
|
+
from llama_index.core.node_parser import SentenceSplitter
|
|
63
|
+
|
|
64
|
+
return SentenceSplitter(
|
|
65
|
+
chunk_size=self.chunk_size,
|
|
66
|
+
chunk_overlap=self.chunk_overlap,
|
|
67
|
+
include_metadata=self.include_metadata,
|
|
68
|
+
include_prev_next_rel=self.include_prev_next_rel,
|
|
69
|
+
)
|
|
70
|
+
if self.chunker_type == "token":
|
|
71
|
+
from llama_index.core.node_parser import TokenTextSplitter
|
|
72
|
+
|
|
73
|
+
return TokenTextSplitter(
|
|
74
|
+
chunk_size=self.chunk_size,
|
|
75
|
+
chunk_overlap=self.chunk_overlap,
|
|
76
|
+
include_metadata=self.include_metadata,
|
|
77
|
+
include_prev_next_rel=self.include_prev_next_rel,
|
|
78
|
+
)
|
|
79
|
+
if self.chunker_type == "fixed":
|
|
80
|
+
from llama_index.core.node_parser import SentenceWindowNodeParser
|
|
81
|
+
|
|
82
|
+
return SentenceWindowNodeParser.from_defaults(
|
|
83
|
+
window_size=self.chunk_size,
|
|
84
|
+
window_metadata_key="window",
|
|
85
|
+
original_text_metadata_key="original_text",
|
|
86
|
+
)
|
|
87
|
+
# markdown as default
|
|
88
|
+
from llama_index.core.node_parser import MarkdownNodeParser
|
|
89
|
+
|
|
90
|
+
return MarkdownNodeParser(
|
|
91
|
+
include_metadata=self.include_metadata,
|
|
92
|
+
include_prev_next_rel=self.include_prev_next_rel,
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
def _find_images_for_chunk(self, doc: Document, chunk_text: str) -> list:
|
|
96
|
+
"""Find images that are referenced in the chunk.
|
|
97
|
+
|
|
98
|
+
Args:
|
|
99
|
+
doc: Original document
|
|
100
|
+
chunk_text: Text of the current chunk
|
|
101
|
+
|
|
102
|
+
Returns:
|
|
103
|
+
List of images referenced in the chunk
|
|
104
|
+
"""
|
|
105
|
+
return [img for img in doc.images if img.filename and img.filename in chunk_text]
|
|
106
|
+
|
|
107
|
+
async def split(
|
|
108
|
+
self,
|
|
109
|
+
doc: Document,
|
|
110
|
+
extra_metadata: dict[str, Any] | None = None,
|
|
111
|
+
) -> list[TextChunk]:
|
|
112
|
+
"""Split document into chunks using LlamaIndex.
|
|
113
|
+
|
|
114
|
+
Args:
|
|
115
|
+
doc: Document to split
|
|
116
|
+
extra_metadata: Additional metadata to include in chunks
|
|
117
|
+
|
|
118
|
+
Returns:
|
|
119
|
+
List of text chunks
|
|
120
|
+
|
|
121
|
+
Raises:
|
|
122
|
+
ImportError: If llama-index isn't installed
|
|
123
|
+
"""
|
|
124
|
+
from llama_index import core
|
|
125
|
+
|
|
126
|
+
meta = {
|
|
127
|
+
"source": doc.source_path or "",
|
|
128
|
+
"title": doc.title or "",
|
|
129
|
+
**(extra_metadata or {}),
|
|
130
|
+
}
|
|
131
|
+
llama_doc = core.Document(text=doc.content, metadata=meta)
|
|
132
|
+
chunker = self._get_llama_chunker()
|
|
133
|
+
nodes = chunker.get_nodes_from_documents([llama_doc])
|
|
134
|
+
chunks = []
|
|
135
|
+
for i, node in enumerate(nodes):
|
|
136
|
+
metadata = {**node.metadata}
|
|
137
|
+
metadata["relationships"] = node.relationships
|
|
138
|
+
chunk_images = self._find_images_for_chunk(doc, node.get_content())
|
|
139
|
+
chunk = TextChunk(
|
|
140
|
+
text=node.get_content(),
|
|
141
|
+
source_doc_id=doc.source_path or "",
|
|
142
|
+
chunk_index=i,
|
|
143
|
+
images=chunk_images,
|
|
144
|
+
metadata={**(extra_metadata or {}), **metadata},
|
|
145
|
+
)
|
|
146
|
+
chunks.append(chunk)
|
|
147
|
+
|
|
148
|
+
return chunks
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
if __name__ == "__main__":
|
|
152
|
+
import asyncio
|
|
153
|
+
|
|
154
|
+
from docler.models import Document
|
|
155
|
+
|
|
156
|
+
async def main():
|
|
157
|
+
# Example usage
|
|
158
|
+
doc = Document(
|
|
159
|
+
source_path="example.txt",
|
|
160
|
+
content="# This is an example\n\nSample content here.",
|
|
161
|
+
)
|
|
162
|
+
chunker = LlamaIndexChunker(chunker_type="markdown")
|
|
163
|
+
chunks = await chunker.split(doc)
|
|
164
|
+
for chunk in chunks:
|
|
165
|
+
print(f"Chunk {chunk.chunk_index}:\n{chunk.text}\n")
|
|
166
|
+
|
|
167
|
+
asyncio.run(main())
|