docler 0.2.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {docler-0.2.0 → docler-0.3.0}/.pre-commit-config.yaml +1 -1
- {docler-0.2.0 → docler-0.3.0}/PKG-INFO +21 -5
- {docler-0.2.0 → docler-0.3.0}/pyproject.toml +14 -5
- {docler-0.2.0 → docler-0.3.0}/src/docler/__init__.py +1 -1
- {docler-0.2.0 → docler-0.3.0}/src/docler/__main__.py +3 -2
- docler-0.3.0/src/docler/annotators/__init__.py +1 -0
- docler-0.3.0/src/docler/annotators/ai_document_annotator.py +125 -0
- docler-0.3.0/src/docler/annotators/ai_image_annotator.py +185 -0
- docler-0.3.0/src/docler/annotators/base.py +34 -0
- docler-0.3.0/src/docler/chunkers/ai_chunker/chunker.py +93 -0
- docler-0.3.0/src/docler/chunkers/ai_chunker/models.py +30 -0
- docler-0.3.0/src/docler/chunkers/ai_chunker/utils.py +40 -0
- docler-0.3.0/src/docler/chunkers/base.py +72 -0
- {docler-0.2.0 → docler-0.3.0}/src/docler/chunkers/llamaindex_chunker/chunker.py +4 -3
- {docler-0.2.0 → docler-0.3.0}/src/docler/chunkers/markdown_chunker/chunker.py +7 -58
- docler-0.3.0/src/docler/chunkers/markdown_chunker/utils.py +58 -0
- {docler-0.2.0 → docler-0.3.0}/src/docler/common_types.py +13 -0
- docler-0.3.0/src/docler/configs/__init__.py +153 -0
- docler-0.3.0/src/docler/configs/annotator_configs.py +123 -0
- docler-0.3.0/src/docler/configs/chunker_configs.py +129 -0
- docler-0.3.0/src/docler/configs/converter_configs.py +342 -0
- docler-0.3.0/src/docler/configs/embedding_configs.py +148 -0
- docler-0.3.0/src/docler/configs/file_db_configs.py +239 -0
- docler-0.3.0/src/docler/configs/processor_configs.py +88 -0
- {docler-0.2.0 → docler-0.3.0}/src/docler/configs/vector_db_configs.py +71 -17
- docler-0.3.0/src/docler/converters/azure_provider/__init__.py +5 -0
- docler-0.2.0/src/docler/converters/azure_provider/converter.py → docler-0.3.0/src/docler/converters/azure_provider/provider.py +59 -90
- {docler-0.2.0 → docler-0.3.0}/src/docler/converters/base.py +9 -1
- {docler-0.2.0 → docler-0.3.0}/src/docler/converters/datalab_provider/provider.py +27 -21
- docler-0.3.0/src/docler/converters/datalab_provider/utils.py +68 -0
- {docler-0.2.0 → docler-0.3.0}/src/docler/converters/docling_provider/provider.py +25 -24
- docler-0.3.0/src/docler/converters/exceptions.py +9 -0
- {docler-0.2.0 → docler-0.3.0}/src/docler/converters/kreuzberg_provider/provider.py +27 -48
- {docler-0.2.0 → docler-0.3.0}/src/docler/converters/llamaparse_provider/provider.py +28 -10
- {docler-0.2.0 → docler-0.3.0}/src/docler/converters/llm_provider/provider.py +12 -18
- {docler-0.2.0 → docler-0.3.0}/src/docler/converters/marker_provider/provider.py +5 -2
- {docler-0.2.0 → docler-0.3.0}/src/docler/converters/markitdown_provider/provider.py +8 -6
- {docler-0.2.0 → docler-0.3.0}/src/docler/converters/mistral_provider/provider.py +9 -9
- docler-0.3.0/src/docler/converters/upstage_provider/__init__.py +5 -0
- docler-0.3.0/src/docler/converters/upstage_provider/provider.py +197 -0
- docler-0.3.0/src/docler/diffs.py +91 -0
- {docler-0.2.0 → docler-0.3.0}/src/docler/embeddings/base.py +2 -8
- {docler-0.2.0 → docler-0.3.0}/src/docler/embeddings/bge_provider.py +5 -2
- {docler-0.2.0 → docler-0.3.0}/src/docler/embeddings/litellm_provider.py +23 -22
- {docler-0.2.0 → docler-0.3.0}/src/docler/embeddings/openai_provider.py +30 -38
- {docler-0.2.0 → docler-0.3.0}/src/docler/embeddings/stf_provider.py +20 -7
- docler-0.3.0/src/docler/log.py +17 -0
- docler-0.3.0/src/docler/mime_types.py +182 -0
- docler-0.3.0/src/docler/models.py +228 -0
- docler-0.3.0/src/docler/processors/ai_processor.py +277 -0
- docler-0.3.0/src/docler/processors/base.py +32 -0
- docler-0.3.0/src/docler/provider.py +70 -0
- docler-0.3.0/src/docler/resources/samplepptx.pptx +0 -0
- docler-0.3.0/src/docler/resources/test.docx +0 -0
- docler-0.3.0/src/docler/resources/test.jpg +0 -0
- docler-0.3.0/src/docler/resources/test.pptx +0 -0
- docler-0.3.0/src/docler/resources/test.xls +0 -0
- docler-0.3.0/src/docler/resources/test.xlsx +0 -0
- {docler-0.2.0 → docler-0.3.0}/src/docler/streamlit_app/app.py +20 -23
- {docler-0.2.0 → docler-0.3.0}/src/docler/streamlit_app/chunkers.py +1 -6
- docler-0.3.0/src/docler/streamlit_app/converters.py +33 -0
- docler-0.3.0/src/docler/streamlit_app/state.py +31 -0
- {docler-0.2.0 → docler-0.3.0}/src/docler/streamlit_app/step1_conversion.py +35 -42
- {docler-0.2.0 → docler-0.3.0}/src/docler/streamlit_app/step2_chunking.py +35 -79
- docler-0.3.0/src/docler/streamlit_app/step3_vectorstore.py +245 -0
- {docler-0.2.0 → docler-0.3.0}/src/docler/streamlit_app/utils.py +5 -10
- {docler-0.2.0 → docler-0.3.0}/src/docler/streamlit_app.py +19 -46
- {docler-0.2.0 → docler-0.3.0}/src/docler/streamlit_chunk_app.py +7 -27
- {docler-0.2.0 → docler-0.3.0}/src/docler/utils.py +65 -3
- docler-0.3.0/src/docler/vector_db/base.py +145 -0
- docler-0.3.0/src/docler/vector_db/base_manager.py +58 -0
- docler-0.3.0/src/docler/vector_db/dbs/chroma_db/__init__.py +6 -0
- {docler-0.2.0 → docler-0.3.0}/src/docler/vector_db/dbs/chroma_db/db.py +15 -69
- docler-0.3.0/src/docler/vector_db/dbs/chroma_db/manager.py +236 -0
- docler-0.3.0/src/docler/vector_db/dbs/openai_db/__init__.py +6 -0
- {docler-0.2.0 → docler-0.3.0}/src/docler/vector_db/dbs/openai_db/db.py +15 -126
- {docler-0.2.0 → docler-0.3.0}/src/docler/vector_db/dbs/openai_db/manager.py +56 -79
- docler-0.3.0/src/docler/vector_db/dbs/openai_db/utils.py +36 -0
- docler-0.3.0/src/docler/vector_db/dbs/pinecone_db/__init__.py +8 -0
- docler-0.3.0/src/docler/vector_db/dbs/pinecone_db/db.py +235 -0
- docler-0.3.0/src/docler/vector_db/dbs/pinecone_db/manager.py +200 -0
- docler-0.3.0/src/docler/vector_db/dbs/pinecone_db/utils.py +82 -0
- docler-0.3.0/src/docler/vector_db/dbs/qdrant_db/db.py +220 -0
- docler-0.3.0/src/docler/vector_db/dbs/qdrant_db/utils.py +21 -0
- docler-0.3.0/tests/test_vector_dbs.py +49 -0
- docler-0.2.0/src/docler/chunkers/ai_chunker/chunker.py +0 -161
- docler-0.2.0/src/docler/chunkers/base.py +0 -55
- docler-0.2.0/src/docler/configs/chunker_configs.py +0 -84
- docler-0.2.0/src/docler/configs/converter_configs.py +0 -206
- docler-0.2.0/src/docler/converters/llamaindex_provider/__init__.py +0 -5
- docler-0.2.0/src/docler/converters/llamaindex_provider/provider.py +0 -124
- docler-0.2.0/src/docler/models.py +0 -62
- docler-0.2.0/src/docler/provider.py +0 -25
- docler-0.2.0/src/docler/streamlit_app/converters.py +0 -23
- docler-0.2.0/src/docler/streamlit_app/state.py +0 -53
- docler-0.2.0/src/docler/vector_db/base.py +0 -447
- docler-0.2.0/src/docler/vector_db/dbs/__init__.py +0 -0
- docler-0.2.0/src/docler/vector_db/dbs/chroma_db/__init__.py +0 -5
- docler-0.2.0/src/docler/vector_db/dbs/openai_db/__init__.py +0 -0
- docler-0.2.0/src/docler/vector_db/dbs/qdrant_db/db.py +0 -334
- docler-0.2.0/src/docler/vector_db/dbs/upstash_db/__init__.py +0 -7
- docler-0.2.0/src/docler/vector_db/dbs/upstash_db/db.py +0 -426
- docler-0.2.0/src/docler/vector_db/dbs/upstash_db/manager.py +0 -298
- {docler-0.2.0 → docler-0.3.0}/.copier-answers.yml +0 -0
- {docler-0.2.0 → docler-0.3.0}/.github/FUNDING.yml +0 -0
- {docler-0.2.0 → docler-0.3.0}/.github/copilot-instructions.md +0 -0
- {docler-0.2.0 → docler-0.3.0}/.github/dependabot.yml +0 -0
- {docler-0.2.0 → docler-0.3.0}/.github/workflows/build.yml +0 -0
- {docler-0.2.0 → docler-0.3.0}/.github/workflows/documentation.yml +0 -0
- {docler-0.2.0 → docler-0.3.0}/.gitignore +0 -0
- {docler-0.2.0 → docler-0.3.0}/LICENSE +0 -0
- {docler-0.2.0 → docler-0.3.0}/README.md +0 -0
- {docler-0.2.0 → docler-0.3.0}/docs/.empty +0 -0
- {docler-0.2.0 → docler-0.3.0}/duties.py +0 -0
- {docler-0.2.0 → docler-0.3.0}/mkdocs.yml +0 -0
- {docler-0.2.0 → docler-0.3.0}/overrides/_dummy.txt +0 -0
- {docler-0.2.0 → docler-0.3.0}/src/docler/chunkers/__init__.py +0 -0
- {docler-0.2.0 → docler-0.3.0}/src/docler/chunkers/ai_chunker/__init__.py +0 -0
- {docler-0.2.0 → docler-0.3.0}/src/docler/chunkers/llamaindex_chunker/__init__.py +0 -0
- {docler-0.2.0 → docler-0.3.0}/src/docler/chunkers/markdown_chunker/__init__.py +0 -0
- {docler-0.2.0/src/docler/configs → docler-0.3.0/src/docler/converters}/__init__.py +0 -0
- {docler-0.2.0 → docler-0.3.0}/src/docler/converters/datalab_provider/__init__.py +0 -0
- {docler-0.2.0 → docler-0.3.0}/src/docler/converters/dir_converter.py +0 -0
- {docler-0.2.0 → docler-0.3.0}/src/docler/converters/docling_provider/__init__.py +0 -0
- {docler-0.2.0 → docler-0.3.0}/src/docler/converters/docling_provider/utils.py +0 -0
- {docler-0.2.0 → docler-0.3.0}/src/docler/converters/kreuzberg_provider/__init__.py +0 -0
- {docler-0.2.0 → docler-0.3.0}/src/docler/converters/llamaparse_provider/__init__.py +0 -0
- {docler-0.2.0 → docler-0.3.0}/src/docler/converters/llm_provider/__init__.py +0 -0
- {docler-0.2.0 → docler-0.3.0}/src/docler/converters/marker_provider/__init__.py +0 -0
- {docler-0.2.0 → docler-0.3.0}/src/docler/converters/markitdown_provider/__init__.py +0 -0
- {docler-0.2.0 → docler-0.3.0}/src/docler/converters/mistral_provider/__init__.py +0 -0
- {docler-0.2.0 → docler-0.3.0}/src/docler/converters/registry.py +0 -0
- {docler-0.2.0 → docler-0.3.0}/src/docler/embeddings/__init__.py +0 -0
- {docler-0.2.0/src/docler/converters → docler-0.3.0/src/docler/processors}/__init__.py +0 -0
- {docler-0.2.0 → docler-0.3.0}/src/docler/py.typed +0 -0
- {docler-0.2.0 → docler-0.3.0}/src/docler/resources/pdf_sample.pdf +0 -0
- {docler-0.2.0 → docler-0.3.0}/src/docler/streamlit_app/__init__.py +0 -0
- {docler-0.2.0 → docler-0.3.0}/src/docler/vector_db/__init__.py +0 -0
- {docler-0.2.0/src/docler/converters/azure_provider → docler-0.3.0/src/docler/vector_db/dbs}/__init__.py +0 -0
- {docler-0.2.0 → docler-0.3.0}/src/docler/vector_db/dbs/qdrant_db/__init__.py +0 -0
- {docler-0.2.0 → docler-0.3.0}/tests/__init__.py +0 -0
- {docler-0.2.0 → docler-0.3.0}/tests/conftest.py +0 -0
- {docler-0.2.0 → docler-0.3.0}/tests/resources/pdf_sample.pdf +0 -0
- {docler-0.2.0 → docler-0.3.0}/tests/test_chunkers.py +0 -0
- {docler-0.2.0 → docler-0.3.0}/tests/test_converters.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: docler
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Abstractions & Tools for OCR / document processing
|
|
5
5
|
Project-URL: Documentation, https://phil65.github.io/docler/
|
|
6
6
|
Project-URL: Source, https://github.com/phil65/docler
|
|
@@ -46,8 +46,10 @@ Classifier: Typing :: Typed
|
|
|
46
46
|
Requires-Python: >=3.12
|
|
47
47
|
Requires-Dist: anyenv>=0.4.1
|
|
48
48
|
Requires-Dist: pydantic
|
|
49
|
+
Requires-Dist: pydantic-settings>=2.8.1
|
|
49
50
|
Requires-Dist: upathtools>=0.4.3
|
|
50
51
|
Provides-Extra: all
|
|
52
|
+
Requires-Dist: azure-ai-documentintelligence; extra == 'all'
|
|
51
53
|
Requires-Dist: chromadb; extra == 'all'
|
|
52
54
|
Requires-Dist: docling; extra == 'all'
|
|
53
55
|
Requires-Dist: kreuzberg; extra == 'all'
|
|
@@ -55,15 +57,23 @@ Requires-Dist: llama-index-readers-smart-pdf-loader; extra == 'all'
|
|
|
55
57
|
Requires-Dist: llmling-agent[pydantic-ai]; extra == 'all'
|
|
56
58
|
Requires-Dist: llmsherpa; extra == 'all'
|
|
57
59
|
Requires-Dist: marker-pdf; extra == 'all'
|
|
58
|
-
Requires-Dist: markitdown; extra == 'all'
|
|
60
|
+
Requires-Dist: markitdown[all]; extra == 'all'
|
|
59
61
|
Requires-Dist: mistralai; extra == 'all'
|
|
60
62
|
Requires-Dist: openai; extra == 'all'
|
|
63
|
+
Requires-Dist: pinecone[asyncio]; extra == 'all'
|
|
64
|
+
Requires-Dist: qdrant-client[fastembed]; extra == 'all'
|
|
61
65
|
Requires-Dist: streamlit; extra == 'all'
|
|
62
66
|
Requires-Dist: upstash-vector; extra == 'all'
|
|
67
|
+
Provides-Extra: azure
|
|
68
|
+
Requires-Dist: azure-ai-documentintelligence; extra == 'azure'
|
|
69
|
+
Provides-Extra: bge
|
|
70
|
+
Requires-Dist: flagembedding; extra == 'bge'
|
|
63
71
|
Provides-Extra: chromadb
|
|
64
72
|
Requires-Dist: chromadb>=0.6.3; extra == 'chromadb'
|
|
73
|
+
Provides-Extra: diffs
|
|
74
|
+
Requires-Dist: diff-match-patch; extra == 'diffs'
|
|
65
75
|
Provides-Extra: docling
|
|
66
|
-
Requires-Dist: docling; extra == 'docling'
|
|
76
|
+
Requires-Dist: docling[ocrmac,rapidocr,vlm]; extra == 'docling'
|
|
67
77
|
Provides-Extra: kreuzberg
|
|
68
78
|
Requires-Dist: kreuzberg; extra == 'kreuzberg'
|
|
69
79
|
Provides-Extra: litellm
|
|
@@ -75,17 +85,23 @@ Provides-Extra: llama-parse
|
|
|
75
85
|
Requires-Dist: llama-parse; extra == 'llama-parse'
|
|
76
86
|
Provides-Extra: marker
|
|
77
87
|
Requires-Dist: marker-pdf; extra == 'marker'
|
|
88
|
+
Provides-Extra: markitdown
|
|
89
|
+
Requires-Dist: markitdown[all]; extra == 'markitdown'
|
|
78
90
|
Provides-Extra: mistralai
|
|
79
91
|
Requires-Dist: mistralai; extra == 'mistralai'
|
|
80
92
|
Provides-Extra: openai
|
|
81
93
|
Requires-Dist: openai; extra == 'openai'
|
|
94
|
+
Provides-Extra: pinecone
|
|
95
|
+
Requires-Dist: pinecone[asyncio]; extra == 'pinecone'
|
|
96
|
+
Provides-Extra: qdrant
|
|
97
|
+
Requires-Dist: qdrant-client[fastembed]; extra == 'qdrant'
|
|
82
98
|
Provides-Extra: smart-pdf
|
|
83
99
|
Requires-Dist: llama-index-readers-smart-pdf-loader; extra == 'smart-pdf'
|
|
84
100
|
Requires-Dist: llmsherpa; extra == 'smart-pdf'
|
|
85
101
|
Provides-Extra: streamlit
|
|
102
|
+
Requires-Dist: streambricks; extra == 'streamlit'
|
|
86
103
|
Requires-Dist: streamlit; extra == 'streamlit'
|
|
87
|
-
|
|
88
|
-
Requires-Dist: upstash-vector; extra == 'upstash'
|
|
104
|
+
Requires-Dist: tokonomics; extra == 'streamlit'
|
|
89
105
|
Description-Content-Type: text/markdown
|
|
90
106
|
|
|
91
107
|
# Docler
|
|
@@ -36,30 +36,36 @@ license = { file = "LICENSE" }
|
|
|
36
36
|
dependencies = [
|
|
37
37
|
"anyenv>=0.4.1",
|
|
38
38
|
"pydantic",
|
|
39
|
+
"pydantic-settings>=2.8.1",
|
|
39
40
|
# Only add below (Copier)
|
|
40
41
|
"upathtools>=0.4.3",
|
|
41
42
|
]
|
|
42
43
|
|
|
44
|
+
|
|
43
45
|
[project.optional-dependencies]
|
|
44
46
|
marker = ["marker-pdf"]
|
|
45
47
|
mistralai = ["mistralai"]
|
|
46
|
-
|
|
48
|
+
diffs = ["diff-match-patch"]
|
|
49
|
+
bge = ["FlagEmbedding"]
|
|
50
|
+
docling = ["docling[ocrmac,vlm,rapidocr]"] # tesserocr
|
|
47
51
|
# olmocr = ["olmocr", "accelerate"]
|
|
48
|
-
|
|
52
|
+
markitdown = ["markitdown[all]"]
|
|
49
53
|
kreuzberg = ["kreuzberg"]
|
|
50
54
|
litellm = ["litellm", "tokonomics"]
|
|
51
|
-
streamlit = ["streamlit"]
|
|
55
|
+
streamlit = ["streamlit", "streambricks", "tokonomics"]
|
|
52
56
|
llama-index = ["llama-index"]
|
|
53
57
|
llama-parse = ["llama-parse"]
|
|
54
58
|
smart-pdf = ["llmsherpa", "llama-index-readers-smart-pdf-loader"]
|
|
55
59
|
chromadb = ["chromadb>=0.6.3"]
|
|
56
|
-
upstash = ["upstash-vector"]
|
|
57
60
|
openai = ["openai"]
|
|
61
|
+
pinecone = ["pinecone[asyncio]"]
|
|
62
|
+
qdrant = ["qdrant-client[fastembed]"]
|
|
63
|
+
azure = ["azure-ai-documentintelligence"]
|
|
58
64
|
all = [
|
|
59
65
|
"marker-pdf",
|
|
60
66
|
"mistralai",
|
|
61
67
|
"docling",
|
|
62
|
-
"markitdown",
|
|
68
|
+
"markitdown[all]",
|
|
63
69
|
"kreuzberg",
|
|
64
70
|
# "litellm",
|
|
65
71
|
"llmling-agent[pydantic-ai]",
|
|
@@ -70,6 +76,9 @@ all = [
|
|
|
70
76
|
"chromadb",
|
|
71
77
|
"upstash-vector",
|
|
72
78
|
"openai",
|
|
79
|
+
"pinecone[asyncio]",
|
|
80
|
+
"qdrant-client[fastembed]",
|
|
81
|
+
"azure-ai-documentintelligence",
|
|
73
82
|
]
|
|
74
83
|
|
|
75
84
|
|
|
@@ -15,7 +15,7 @@ from docler.converters.llm_provider import LLMConverter
|
|
|
15
15
|
from docler.converters.datalab_provider import DataLabConverter
|
|
16
16
|
from docler.converters.llamaparse_provider import LlamaParseConverter
|
|
17
17
|
|
|
18
|
-
__version__ = "0.
|
|
18
|
+
__version__ = "0.3.0"
|
|
19
19
|
|
|
20
20
|
__all__ = [
|
|
21
21
|
"Conversion",
|
|
@@ -2,7 +2,6 @@
|
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
-
import logging
|
|
6
5
|
import os
|
|
7
6
|
import pathlib
|
|
8
7
|
import subprocess
|
|
@@ -10,10 +9,12 @@ import sys
|
|
|
10
9
|
|
|
11
10
|
import typer
|
|
12
11
|
|
|
12
|
+
from docler.log import get_logger
|
|
13
|
+
|
|
13
14
|
|
|
14
15
|
cli = typer.Typer(help="Docler document converter CLI", no_args_is_help=True)
|
|
15
16
|
|
|
16
|
-
logger =
|
|
17
|
+
logger = get_logger(__name__)
|
|
17
18
|
|
|
18
19
|
PACKAGE_DIR = pathlib.Path(__file__).parent
|
|
19
20
|
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Annotators for Documents."""
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
"""AI-powered document and chunk metadata annotation."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
from itertools import batched
|
|
7
|
+
from typing import TYPE_CHECKING, ClassVar, TypeVar
|
|
8
|
+
|
|
9
|
+
from pydantic import BaseModel
|
|
10
|
+
|
|
11
|
+
from docler.annotators.base import Annotator
|
|
12
|
+
from docler.common_types import DEFAULT_ANNOTATOR_MODEL
|
|
13
|
+
from docler.configs.annotator_configs import (
|
|
14
|
+
DEFAULT_DOC_PROMPT_TEMPLATE,
|
|
15
|
+
DEFAULT_DOC_SYSTEM_PROMPT,
|
|
16
|
+
AIDocumentAnnotatorConfig,
|
|
17
|
+
)
|
|
18
|
+
from docler.log import get_logger
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
if TYPE_CHECKING:
|
|
22
|
+
from docler.models import ChunkedDocument
|
|
23
|
+
|
|
24
|
+
logger = get_logger(__name__)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class DefaultMetadata(BaseModel):
|
|
28
|
+
"""Default metadata for a document or chunk."""
|
|
29
|
+
|
|
30
|
+
topics: list[str]
|
|
31
|
+
"""Topics/categories."""
|
|
32
|
+
|
|
33
|
+
keywords: list[str]
|
|
34
|
+
"""Keywords."""
|
|
35
|
+
|
|
36
|
+
entities: list[str]
|
|
37
|
+
"""Main entities."""
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
# Type variable for generic metadata model
|
|
41
|
+
T = TypeVar("T", bound=BaseModel)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class AIDocumentAnnotator[TMetadata](Annotator[AIDocumentAnnotatorConfig]):
|
|
45
|
+
"""AI-based document and chunk annotator.
|
|
46
|
+
|
|
47
|
+
Enhances documents and chunks with metadata.
|
|
48
|
+
|
|
49
|
+
Type Parameters:
|
|
50
|
+
T: Type of metadata model to use. Must be a Pydantic BaseModel.
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
Config = AIDocumentAnnotatorConfig
|
|
54
|
+
REQUIRED_PACKAGES: ClassVar = {"llmling-agent"}
|
|
55
|
+
|
|
56
|
+
def __init__(
|
|
57
|
+
self,
|
|
58
|
+
model: str | None = None,
|
|
59
|
+
system_prompt: str | None = None,
|
|
60
|
+
user_prompt: str | None = None,
|
|
61
|
+
metadata_model: type[TMetadata] = DefaultMetadata, # type: ignore
|
|
62
|
+
max_context_length: int = 1500,
|
|
63
|
+
batch_size: int = 5,
|
|
64
|
+
):
|
|
65
|
+
"""Initialize the AI document annotator.
|
|
66
|
+
|
|
67
|
+
Args:
|
|
68
|
+
model: LLM model to use for annotation
|
|
69
|
+
system_prompt: Optional custom prompt for annotation
|
|
70
|
+
user_prompt: Optional custom prompt for annotation
|
|
71
|
+
metadata_model: Pydantic model class for metadata structure
|
|
72
|
+
max_context_length: Maximum length of context for annotation
|
|
73
|
+
batch_size: Number of chunks to process in parallel
|
|
74
|
+
"""
|
|
75
|
+
self.model = model or DEFAULT_ANNOTATOR_MODEL
|
|
76
|
+
self.system_prompt = system_prompt or DEFAULT_DOC_SYSTEM_PROMPT
|
|
77
|
+
self.user_prompt = user_prompt or DEFAULT_DOC_PROMPT_TEMPLATE
|
|
78
|
+
self.metadata_model = metadata_model
|
|
79
|
+
self.max_context_length = max_context_length
|
|
80
|
+
self.batch_size = batch_size
|
|
81
|
+
|
|
82
|
+
async def annotate(self, document: ChunkedDocument) -> ChunkedDocument:
|
|
83
|
+
"""Annotate document and chunks with AI-generated metadata.
|
|
84
|
+
|
|
85
|
+
Args:
|
|
86
|
+
document: Chunked document to annotate
|
|
87
|
+
|
|
88
|
+
Returns:
|
|
89
|
+
Document with enhanced metadata
|
|
90
|
+
"""
|
|
91
|
+
from llmling_agent import Agent, StructuredAgent
|
|
92
|
+
|
|
93
|
+
agent: StructuredAgent[None, TMetadata] = Agent[None]( # type: ignore
|
|
94
|
+
model=self.model,
|
|
95
|
+
system_prompt=self.system_prompt,
|
|
96
|
+
).to_structured(self.metadata_model)
|
|
97
|
+
|
|
98
|
+
# Process document-level metadata if needed
|
|
99
|
+
if document.metadata is None:
|
|
100
|
+
document.metadata = {}
|
|
101
|
+
|
|
102
|
+
# Get a condensed version of the document for context
|
|
103
|
+
context = (
|
|
104
|
+
document.content[: self.max_context_length] + "..."
|
|
105
|
+
if self.max_context_length and len(document.content) > self.max_context_length
|
|
106
|
+
else document.content
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
# Process chunks in batches
|
|
110
|
+
for batch in batched(document.chunks, self.batch_size):
|
|
111
|
+
tasks = []
|
|
112
|
+
|
|
113
|
+
for chunk in batch:
|
|
114
|
+
prompt = self.user_prompt.format(context=context, chunk=chunk.text)
|
|
115
|
+
tasks.append(agent.run(prompt))
|
|
116
|
+
|
|
117
|
+
try:
|
|
118
|
+
results = await asyncio.gather(*tasks)
|
|
119
|
+
for chunk, result in zip(batch, results):
|
|
120
|
+
metadata = result.content.model_dump() # type: ignore
|
|
121
|
+
chunk.metadata |= metadata
|
|
122
|
+
except Exception:
|
|
123
|
+
logger.exception("Error annotating batch")
|
|
124
|
+
|
|
125
|
+
return document
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
"""AI-powered image annotation."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import base64
|
|
7
|
+
from itertools import batched
|
|
8
|
+
from typing import TYPE_CHECKING, ClassVar
|
|
9
|
+
|
|
10
|
+
from pydantic import BaseModel
|
|
11
|
+
from typing_extensions import TypeVar
|
|
12
|
+
|
|
13
|
+
from docler.annotators.base import Annotator
|
|
14
|
+
from docler.common_types import DEFAULT_IMAGE_ANNOTATOR_MODEL
|
|
15
|
+
from docler.configs.annotator_configs import (
|
|
16
|
+
DEFAULT_IMAGE_PROMPT_TEMPLATE,
|
|
17
|
+
DEFAULT_IMAGE_SYSTEM_PROMPT,
|
|
18
|
+
AIImageAnnotatorConfig,
|
|
19
|
+
)
|
|
20
|
+
from docler.log import get_logger
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
if TYPE_CHECKING:
|
|
24
|
+
from docler.models import ChunkedDocument, Image
|
|
25
|
+
|
|
26
|
+
logger = get_logger(__name__)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class DefaultImageMetadata(BaseModel):
|
|
30
|
+
"""Default metadata for an image."""
|
|
31
|
+
|
|
32
|
+
description: str
|
|
33
|
+
"""Detailed description of the image."""
|
|
34
|
+
|
|
35
|
+
objects: list[str]
|
|
36
|
+
"""Objects identified in the image."""
|
|
37
|
+
|
|
38
|
+
text_content: str | None = None
|
|
39
|
+
"""Text visible in the image, if any."""
|
|
40
|
+
|
|
41
|
+
image_type: str
|
|
42
|
+
"""Type of image (photo, diagram, chart, etc.)."""
|
|
43
|
+
|
|
44
|
+
colors: list[str]
|
|
45
|
+
"""Dominant colors in the image."""
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
TMetadata = TypeVar("TMetadata", bound=BaseModel, default=DefaultImageMetadata)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class AIImageAnnotator[TMetadata](Annotator[AIImageAnnotatorConfig]):
|
|
52
|
+
"""AI-based image annotator.
|
|
53
|
+
|
|
54
|
+
Analyzes images in chunks and adds descriptions and metadata.
|
|
55
|
+
|
|
56
|
+
Type Parameters:
|
|
57
|
+
TMetadata: Type of metadata model to use. Must be a Pydantic BaseModel.
|
|
58
|
+
"""
|
|
59
|
+
|
|
60
|
+
Config = AIImageAnnotatorConfig
|
|
61
|
+
|
|
62
|
+
REQUIRED_PACKAGES: ClassVar = {"llmling-agent"}
|
|
63
|
+
|
|
64
|
+
def __init__(
|
|
65
|
+
self,
|
|
66
|
+
model: str | None = None,
|
|
67
|
+
system_prompt: str | None = None,
|
|
68
|
+
user_prompt: str | None = None,
|
|
69
|
+
metadata_model: type[TMetadata] = DefaultImageMetadata, # type: ignore
|
|
70
|
+
batch_size: int = 3,
|
|
71
|
+
):
|
|
72
|
+
"""Initialize the AI image annotator.
|
|
73
|
+
|
|
74
|
+
Args:
|
|
75
|
+
model: Vision model to use (must support images)
|
|
76
|
+
system_prompt: Custom prompt for image analysis
|
|
77
|
+
user_prompt: Custom user prompt template for image analysis
|
|
78
|
+
metadata_model: Pydantic model for image metadata
|
|
79
|
+
batch_size: Number of images to process concurrently
|
|
80
|
+
"""
|
|
81
|
+
self.model = model or DEFAULT_IMAGE_ANNOTATOR_MODEL
|
|
82
|
+
self.system_prompt = system_prompt or DEFAULT_IMAGE_SYSTEM_PROMPT
|
|
83
|
+
self.user_prompt = user_prompt or DEFAULT_IMAGE_PROMPT_TEMPLATE
|
|
84
|
+
self.metadata_model = metadata_model
|
|
85
|
+
self.batch_size = batch_size
|
|
86
|
+
|
|
87
|
+
async def _process_image(self, image: Image) -> Image:
|
|
88
|
+
"""Process a single image with the vision model.
|
|
89
|
+
|
|
90
|
+
Args:
|
|
91
|
+
image: Image to analyze
|
|
92
|
+
|
|
93
|
+
Returns:
|
|
94
|
+
Image with added description and metadata
|
|
95
|
+
"""
|
|
96
|
+
from llmling_agent import Agent, ImageBase64Content, StructuredAgent
|
|
97
|
+
|
|
98
|
+
if image.description and image.metadata:
|
|
99
|
+
return image
|
|
100
|
+
|
|
101
|
+
if isinstance(image.content, bytes):
|
|
102
|
+
b64_content = base64.b64encode(image.content).decode("utf-8")
|
|
103
|
+
else:
|
|
104
|
+
b64_content = (
|
|
105
|
+
image.content
|
|
106
|
+
if not image.content.startswith("data:")
|
|
107
|
+
else image.content.split(",", 1)[1]
|
|
108
|
+
)
|
|
109
|
+
img_content = ImageBase64Content(data=b64_content, mime_type=image.mime_type)
|
|
110
|
+
agent: StructuredAgent[None, TMetadata] = Agent[None]( # type: ignore
|
|
111
|
+
model=self.model,
|
|
112
|
+
system_prompt=self.system_prompt,
|
|
113
|
+
).to_structured(self.metadata_model)
|
|
114
|
+
|
|
115
|
+
try:
|
|
116
|
+
filename_info = f" ({image.filename})" if image.filename else ""
|
|
117
|
+
prompt = self.user_prompt.format(
|
|
118
|
+
image_id=image.id,
|
|
119
|
+
filename_info=filename_info,
|
|
120
|
+
filename=image.filename or "",
|
|
121
|
+
mime_type=image.mime_type,
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
result = await agent.run(prompt, img_content)
|
|
125
|
+
metadata = result.content.model_dump() # type: ignore
|
|
126
|
+
description = metadata.pop("description", None)
|
|
127
|
+
if description:
|
|
128
|
+
image.description = description
|
|
129
|
+
if image.metadata is None:
|
|
130
|
+
image.metadata = {}
|
|
131
|
+
image.metadata.update(metadata)
|
|
132
|
+
|
|
133
|
+
except Exception:
|
|
134
|
+
logger.exception("Error processing image %s", image.id)
|
|
135
|
+
|
|
136
|
+
return image
|
|
137
|
+
|
|
138
|
+
async def annotate(self, document: ChunkedDocument) -> ChunkedDocument:
|
|
139
|
+
"""Annotate all images in the chunks with AI-generated descriptions.
|
|
140
|
+
|
|
141
|
+
Args:
|
|
142
|
+
document: Chunked document containing chunks with images to annotate
|
|
143
|
+
|
|
144
|
+
Returns:
|
|
145
|
+
Document with annotated images in chunks
|
|
146
|
+
"""
|
|
147
|
+
for chunk in document.chunks:
|
|
148
|
+
if not chunk.images:
|
|
149
|
+
continue
|
|
150
|
+
|
|
151
|
+
for batch in batched(chunk.images, self.batch_size):
|
|
152
|
+
tasks = [self._process_image(img) for img in batch]
|
|
153
|
+
try:
|
|
154
|
+
await asyncio.gather(*tasks)
|
|
155
|
+
except Exception:
|
|
156
|
+
msg = "Error processing images in chunk %s"
|
|
157
|
+
logger.exception(msg, chunk.chunk_index)
|
|
158
|
+
|
|
159
|
+
return document
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
if __name__ == "__main__":
|
|
163
|
+
import asyncio
|
|
164
|
+
|
|
165
|
+
from docler.models import ChunkedDocument, Image, TextChunk
|
|
166
|
+
|
|
167
|
+
async def main():
|
|
168
|
+
annotator = AIImageAnnotator[DefaultImageMetadata]()
|
|
169
|
+
url = "https://www.a-i-stack.com/wp-content/uploads/go-x/u/93dcedb9-17f3-4aee-9b5a-3744e5e84686/image-342x342.png"
|
|
170
|
+
image = await Image.from_file(url)
|
|
171
|
+
document = ChunkedDocument(
|
|
172
|
+
content="test",
|
|
173
|
+
chunks=[
|
|
174
|
+
TextChunk(
|
|
175
|
+
text="Sample text",
|
|
176
|
+
source_doc_id="sample_doc_id",
|
|
177
|
+
images=[image],
|
|
178
|
+
chunk_index=0,
|
|
179
|
+
),
|
|
180
|
+
],
|
|
181
|
+
)
|
|
182
|
+
doc = await annotator.annotate(document)
|
|
183
|
+
print(doc)
|
|
184
|
+
|
|
185
|
+
asyncio.run(main())
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""Base classes for text chunking implementations."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from abc import ABC, abstractmethod
|
|
6
|
+
from typing import TYPE_CHECKING, ClassVar
|
|
7
|
+
|
|
8
|
+
from docler.provider import BaseProvider
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
if TYPE_CHECKING:
|
|
12
|
+
from docler.configs.annotator_configs import BaseAnnotatorConfig
|
|
13
|
+
from docler.models import ChunkedDocument
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class Annotator[TConfig](ABC, BaseProvider[TConfig]):
|
|
17
|
+
"""Base class for chunk annotation processors."""
|
|
18
|
+
|
|
19
|
+
Config: ClassVar[type[BaseAnnotatorConfig]]
|
|
20
|
+
|
|
21
|
+
@abstractmethod
|
|
22
|
+
async def annotate(
|
|
23
|
+
self,
|
|
24
|
+
chunked_doc: ChunkedDocument,
|
|
25
|
+
) -> ChunkedDocument:
|
|
26
|
+
"""Annotate a chunked document with additional metadata.
|
|
27
|
+
|
|
28
|
+
Args:
|
|
29
|
+
chunked_doc: Chunked document containing the original content and chunks
|
|
30
|
+
|
|
31
|
+
Returns:
|
|
32
|
+
Chunked document with annotated chunks
|
|
33
|
+
"""
|
|
34
|
+
raise NotImplementedError
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
"""AI-based markdown chunking implementation."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import TYPE_CHECKING, Any, ClassVar
|
|
6
|
+
|
|
7
|
+
from docler.chunkers.ai_chunker.models import Chunk, Chunks
|
|
8
|
+
from docler.chunkers.ai_chunker.utils import add_line_numbers, create_text_chunk
|
|
9
|
+
from docler.chunkers.base import TextChunker
|
|
10
|
+
from docler.common_types import DEFAULT_CHUNKER_MODEL
|
|
11
|
+
from docler.configs.chunker_configs import (
|
|
12
|
+
DEFAULT_CHUNKER_SYSTEM_PROMPT,
|
|
13
|
+
DEFAULT_CHUNKER_USER_TEMPLATE,
|
|
14
|
+
)
|
|
15
|
+
from docler.models import TextChunk
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
if TYPE_CHECKING:
|
|
19
|
+
from docler.models import Document
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class AIChunker(TextChunker):
|
|
23
|
+
"""LLM-based document chunker."""
|
|
24
|
+
|
|
25
|
+
REQUIRED_PACKAGES: ClassVar = {"llmling-agent"}
|
|
26
|
+
|
|
27
|
+
def __init__(
|
|
28
|
+
self,
|
|
29
|
+
model: str | None = None,
|
|
30
|
+
user_prompt: str | None = None,
|
|
31
|
+
system_prompt: str | None = None,
|
|
32
|
+
):
|
|
33
|
+
"""Initialize the AI chunker.
|
|
34
|
+
|
|
35
|
+
Args:
|
|
36
|
+
model: LLM model to use
|
|
37
|
+
provider: LLM provider to use
|
|
38
|
+
user_prompt: User prompt to use
|
|
39
|
+
system_prompt: System prompt to use
|
|
40
|
+
"""
|
|
41
|
+
self.model = model or DEFAULT_CHUNKER_MODEL
|
|
42
|
+
self.user_prompt = user_prompt or DEFAULT_CHUNKER_USER_TEMPLATE
|
|
43
|
+
self.system_prompt = system_prompt or DEFAULT_CHUNKER_SYSTEM_PROMPT
|
|
44
|
+
|
|
45
|
+
async def _get_chunks(self, text: str) -> Chunks:
|
|
46
|
+
"""Get chunk definitions from LLM."""
|
|
47
|
+
from llmling_agent import Agent
|
|
48
|
+
|
|
49
|
+
numbered_text = add_line_numbers(text)
|
|
50
|
+
# agent: llmling_agent.StructuredAgent[None, Chunks] = llmling_agent.Agent[None](
|
|
51
|
+
# model=self.model,
|
|
52
|
+
# system_prompt=self.system_prompt,
|
|
53
|
+
# ).to_structured(Chunks)
|
|
54
|
+
# prompt = CHUNKING_PROMPT.format(numbered_text=numbered_text)
|
|
55
|
+
# response = await agent.run(prompt)
|
|
56
|
+
agent: Agent[None] = Agent[None](
|
|
57
|
+
model=self.model, system_prompt=self.system_prompt
|
|
58
|
+
)
|
|
59
|
+
prompt = self.user_prompt.format(numbered_text=numbered_text)
|
|
60
|
+
chunks = await agent.talk.extract_multiple(
|
|
61
|
+
text,
|
|
62
|
+
Chunk,
|
|
63
|
+
prompt=prompt,
|
|
64
|
+
mode="structured", # tool_calls
|
|
65
|
+
)
|
|
66
|
+
return Chunks(chunks=chunks)
|
|
67
|
+
|
|
68
|
+
async def split(
|
|
69
|
+
self,
|
|
70
|
+
doc: Document,
|
|
71
|
+
extra_metadata: dict[str, Any] | None = None,
|
|
72
|
+
) -> list[TextChunk]:
|
|
73
|
+
"""Split document into chunks using LLM analysis."""
|
|
74
|
+
chunks = await self._get_chunks(doc.content)
|
|
75
|
+
return [
|
|
76
|
+
create_text_chunk(doc, chunk, i, extra_metadata)
|
|
77
|
+
for i, chunk in enumerate(chunks.chunks)
|
|
78
|
+
]
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
if __name__ == "__main__":
|
|
82
|
+
import asyncio
|
|
83
|
+
|
|
84
|
+
from docler.models import Document
|
|
85
|
+
|
|
86
|
+
async def main():
|
|
87
|
+
# Example usage
|
|
88
|
+
doc = Document(source_path="example.txt", content=DEFAULT_CHUNKER_SYSTEM_PROMPT)
|
|
89
|
+
chunker = AIChunker()
|
|
90
|
+
chunks = await chunker.split(doc)
|
|
91
|
+
print(chunks)
|
|
92
|
+
|
|
93
|
+
asyncio.run(main())
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""Models for AI chunker."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from pydantic import BaseModel, ConfigDict
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class Chunk(BaseModel):
|
|
9
|
+
"""A chunk of text with semantic metadata."""
|
|
10
|
+
|
|
11
|
+
start_row: int
|
|
12
|
+
"""Start line number (1-based)"""
|
|
13
|
+
|
|
14
|
+
end_row: int
|
|
15
|
+
"""End line number (1-based)"""
|
|
16
|
+
|
|
17
|
+
keywords: list[str]
|
|
18
|
+
"""Key terms and concepts in this chunk"""
|
|
19
|
+
|
|
20
|
+
references: list[int]
|
|
21
|
+
"""Line numbers that this chunk references or depends on"""
|
|
22
|
+
|
|
23
|
+
model_config = ConfigDict(use_attribute_docstrings=True)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class Chunks(BaseModel):
|
|
27
|
+
"""Collection of chunks with their metadata."""
|
|
28
|
+
|
|
29
|
+
chunks: list[Chunk]
|
|
30
|
+
"""A list of chunks to extract from the document."""
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"""AI-based markdown chunking implementation."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import TYPE_CHECKING, Any
|
|
6
|
+
|
|
7
|
+
from docler.models import TextChunk
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
if TYPE_CHECKING:
|
|
11
|
+
from docler.chunkers.ai_chunker.models import Chunk
|
|
12
|
+
from docler.models import Document
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def add_line_numbers(text: str) -> str:
|
|
16
|
+
"""Add line numbers to text."""
|
|
17
|
+
lines = text.splitlines()
|
|
18
|
+
return "\n".join(f"{i + 1:3d} | {line}" for i, line in enumerate(lines))
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def create_text_chunk(
|
|
22
|
+
doc: Document,
|
|
23
|
+
chunk: Chunk,
|
|
24
|
+
chunk_idx: int,
|
|
25
|
+
extra_metadata: dict[str, Any] | None = None,
|
|
26
|
+
) -> TextChunk:
|
|
27
|
+
"""Create a TextChunk from chunk definition."""
|
|
28
|
+
lines = doc.content.splitlines()
|
|
29
|
+
chunk_lines = lines[chunk.start_row - 1 : chunk.end_row]
|
|
30
|
+
chunk_text = "\n".join(chunk_lines)
|
|
31
|
+
base = extra_metadata or {}
|
|
32
|
+
metadata = {**base, "keywords": chunk.keywords, "references": chunk.references}
|
|
33
|
+
chunk_images = [i for i in doc.images if i.filename and i.filename in chunk_text]
|
|
34
|
+
return TextChunk(
|
|
35
|
+
text=chunk_text,
|
|
36
|
+
source_doc_id=doc.source_path or "",
|
|
37
|
+
chunk_index=chunk_idx,
|
|
38
|
+
images=chunk_images,
|
|
39
|
+
metadata=metadata,
|
|
40
|
+
)
|