docler 0.2.0__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (145) hide show
  1. {docler-0.2.0 → docler-0.3.0}/.pre-commit-config.yaml +1 -1
  2. {docler-0.2.0 → docler-0.3.0}/PKG-INFO +21 -5
  3. {docler-0.2.0 → docler-0.3.0}/pyproject.toml +14 -5
  4. {docler-0.2.0 → docler-0.3.0}/src/docler/__init__.py +1 -1
  5. {docler-0.2.0 → docler-0.3.0}/src/docler/__main__.py +3 -2
  6. docler-0.3.0/src/docler/annotators/__init__.py +1 -0
  7. docler-0.3.0/src/docler/annotators/ai_document_annotator.py +125 -0
  8. docler-0.3.0/src/docler/annotators/ai_image_annotator.py +185 -0
  9. docler-0.3.0/src/docler/annotators/base.py +34 -0
  10. docler-0.3.0/src/docler/chunkers/ai_chunker/chunker.py +93 -0
  11. docler-0.3.0/src/docler/chunkers/ai_chunker/models.py +30 -0
  12. docler-0.3.0/src/docler/chunkers/ai_chunker/utils.py +40 -0
  13. docler-0.3.0/src/docler/chunkers/base.py +72 -0
  14. {docler-0.2.0 → docler-0.3.0}/src/docler/chunkers/llamaindex_chunker/chunker.py +4 -3
  15. {docler-0.2.0 → docler-0.3.0}/src/docler/chunkers/markdown_chunker/chunker.py +7 -58
  16. docler-0.3.0/src/docler/chunkers/markdown_chunker/utils.py +58 -0
  17. {docler-0.2.0 → docler-0.3.0}/src/docler/common_types.py +13 -0
  18. docler-0.3.0/src/docler/configs/__init__.py +153 -0
  19. docler-0.3.0/src/docler/configs/annotator_configs.py +123 -0
  20. docler-0.3.0/src/docler/configs/chunker_configs.py +129 -0
  21. docler-0.3.0/src/docler/configs/converter_configs.py +342 -0
  22. docler-0.3.0/src/docler/configs/embedding_configs.py +148 -0
  23. docler-0.3.0/src/docler/configs/file_db_configs.py +239 -0
  24. docler-0.3.0/src/docler/configs/processor_configs.py +88 -0
  25. {docler-0.2.0 → docler-0.3.0}/src/docler/configs/vector_db_configs.py +71 -17
  26. docler-0.3.0/src/docler/converters/azure_provider/__init__.py +5 -0
  27. docler-0.2.0/src/docler/converters/azure_provider/converter.py → docler-0.3.0/src/docler/converters/azure_provider/provider.py +59 -90
  28. {docler-0.2.0 → docler-0.3.0}/src/docler/converters/base.py +9 -1
  29. {docler-0.2.0 → docler-0.3.0}/src/docler/converters/datalab_provider/provider.py +27 -21
  30. docler-0.3.0/src/docler/converters/datalab_provider/utils.py +68 -0
  31. {docler-0.2.0 → docler-0.3.0}/src/docler/converters/docling_provider/provider.py +25 -24
  32. docler-0.3.0/src/docler/converters/exceptions.py +9 -0
  33. {docler-0.2.0 → docler-0.3.0}/src/docler/converters/kreuzberg_provider/provider.py +27 -48
  34. {docler-0.2.0 → docler-0.3.0}/src/docler/converters/llamaparse_provider/provider.py +28 -10
  35. {docler-0.2.0 → docler-0.3.0}/src/docler/converters/llm_provider/provider.py +12 -18
  36. {docler-0.2.0 → docler-0.3.0}/src/docler/converters/marker_provider/provider.py +5 -2
  37. {docler-0.2.0 → docler-0.3.0}/src/docler/converters/markitdown_provider/provider.py +8 -6
  38. {docler-0.2.0 → docler-0.3.0}/src/docler/converters/mistral_provider/provider.py +9 -9
  39. docler-0.3.0/src/docler/converters/upstage_provider/__init__.py +5 -0
  40. docler-0.3.0/src/docler/converters/upstage_provider/provider.py +197 -0
  41. docler-0.3.0/src/docler/diffs.py +91 -0
  42. {docler-0.2.0 → docler-0.3.0}/src/docler/embeddings/base.py +2 -8
  43. {docler-0.2.0 → docler-0.3.0}/src/docler/embeddings/bge_provider.py +5 -2
  44. {docler-0.2.0 → docler-0.3.0}/src/docler/embeddings/litellm_provider.py +23 -22
  45. {docler-0.2.0 → docler-0.3.0}/src/docler/embeddings/openai_provider.py +30 -38
  46. {docler-0.2.0 → docler-0.3.0}/src/docler/embeddings/stf_provider.py +20 -7
  47. docler-0.3.0/src/docler/log.py +17 -0
  48. docler-0.3.0/src/docler/mime_types.py +182 -0
  49. docler-0.3.0/src/docler/models.py +228 -0
  50. docler-0.3.0/src/docler/processors/ai_processor.py +277 -0
  51. docler-0.3.0/src/docler/processors/base.py +32 -0
  52. docler-0.3.0/src/docler/provider.py +70 -0
  53. docler-0.3.0/src/docler/resources/samplepptx.pptx +0 -0
  54. docler-0.3.0/src/docler/resources/test.docx +0 -0
  55. docler-0.3.0/src/docler/resources/test.jpg +0 -0
  56. docler-0.3.0/src/docler/resources/test.pptx +0 -0
  57. docler-0.3.0/src/docler/resources/test.xls +0 -0
  58. docler-0.3.0/src/docler/resources/test.xlsx +0 -0
  59. {docler-0.2.0 → docler-0.3.0}/src/docler/streamlit_app/app.py +20 -23
  60. {docler-0.2.0 → docler-0.3.0}/src/docler/streamlit_app/chunkers.py +1 -6
  61. docler-0.3.0/src/docler/streamlit_app/converters.py +33 -0
  62. docler-0.3.0/src/docler/streamlit_app/state.py +31 -0
  63. {docler-0.2.0 → docler-0.3.0}/src/docler/streamlit_app/step1_conversion.py +35 -42
  64. {docler-0.2.0 → docler-0.3.0}/src/docler/streamlit_app/step2_chunking.py +35 -79
  65. docler-0.3.0/src/docler/streamlit_app/step3_vectorstore.py +245 -0
  66. {docler-0.2.0 → docler-0.3.0}/src/docler/streamlit_app/utils.py +5 -10
  67. {docler-0.2.0 → docler-0.3.0}/src/docler/streamlit_app.py +19 -46
  68. {docler-0.2.0 → docler-0.3.0}/src/docler/streamlit_chunk_app.py +7 -27
  69. {docler-0.2.0 → docler-0.3.0}/src/docler/utils.py +65 -3
  70. docler-0.3.0/src/docler/vector_db/base.py +145 -0
  71. docler-0.3.0/src/docler/vector_db/base_manager.py +58 -0
  72. docler-0.3.0/src/docler/vector_db/dbs/chroma_db/__init__.py +6 -0
  73. {docler-0.2.0 → docler-0.3.0}/src/docler/vector_db/dbs/chroma_db/db.py +15 -69
  74. docler-0.3.0/src/docler/vector_db/dbs/chroma_db/manager.py +236 -0
  75. docler-0.3.0/src/docler/vector_db/dbs/openai_db/__init__.py +6 -0
  76. {docler-0.2.0 → docler-0.3.0}/src/docler/vector_db/dbs/openai_db/db.py +15 -126
  77. {docler-0.2.0 → docler-0.3.0}/src/docler/vector_db/dbs/openai_db/manager.py +56 -79
  78. docler-0.3.0/src/docler/vector_db/dbs/openai_db/utils.py +36 -0
  79. docler-0.3.0/src/docler/vector_db/dbs/pinecone_db/__init__.py +8 -0
  80. docler-0.3.0/src/docler/vector_db/dbs/pinecone_db/db.py +235 -0
  81. docler-0.3.0/src/docler/vector_db/dbs/pinecone_db/manager.py +200 -0
  82. docler-0.3.0/src/docler/vector_db/dbs/pinecone_db/utils.py +82 -0
  83. docler-0.3.0/src/docler/vector_db/dbs/qdrant_db/db.py +220 -0
  84. docler-0.3.0/src/docler/vector_db/dbs/qdrant_db/utils.py +21 -0
  85. docler-0.3.0/tests/test_vector_dbs.py +49 -0
  86. docler-0.2.0/src/docler/chunkers/ai_chunker/chunker.py +0 -161
  87. docler-0.2.0/src/docler/chunkers/base.py +0 -55
  88. docler-0.2.0/src/docler/configs/chunker_configs.py +0 -84
  89. docler-0.2.0/src/docler/configs/converter_configs.py +0 -206
  90. docler-0.2.0/src/docler/converters/llamaindex_provider/__init__.py +0 -5
  91. docler-0.2.0/src/docler/converters/llamaindex_provider/provider.py +0 -124
  92. docler-0.2.0/src/docler/models.py +0 -62
  93. docler-0.2.0/src/docler/provider.py +0 -25
  94. docler-0.2.0/src/docler/streamlit_app/converters.py +0 -23
  95. docler-0.2.0/src/docler/streamlit_app/state.py +0 -53
  96. docler-0.2.0/src/docler/vector_db/base.py +0 -447
  97. docler-0.2.0/src/docler/vector_db/dbs/__init__.py +0 -0
  98. docler-0.2.0/src/docler/vector_db/dbs/chroma_db/__init__.py +0 -5
  99. docler-0.2.0/src/docler/vector_db/dbs/openai_db/__init__.py +0 -0
  100. docler-0.2.0/src/docler/vector_db/dbs/qdrant_db/db.py +0 -334
  101. docler-0.2.0/src/docler/vector_db/dbs/upstash_db/__init__.py +0 -7
  102. docler-0.2.0/src/docler/vector_db/dbs/upstash_db/db.py +0 -426
  103. docler-0.2.0/src/docler/vector_db/dbs/upstash_db/manager.py +0 -298
  104. {docler-0.2.0 → docler-0.3.0}/.copier-answers.yml +0 -0
  105. {docler-0.2.0 → docler-0.3.0}/.github/FUNDING.yml +0 -0
  106. {docler-0.2.0 → docler-0.3.0}/.github/copilot-instructions.md +0 -0
  107. {docler-0.2.0 → docler-0.3.0}/.github/dependabot.yml +0 -0
  108. {docler-0.2.0 → docler-0.3.0}/.github/workflows/build.yml +0 -0
  109. {docler-0.2.0 → docler-0.3.0}/.github/workflows/documentation.yml +0 -0
  110. {docler-0.2.0 → docler-0.3.0}/.gitignore +0 -0
  111. {docler-0.2.0 → docler-0.3.0}/LICENSE +0 -0
  112. {docler-0.2.0 → docler-0.3.0}/README.md +0 -0
  113. {docler-0.2.0 → docler-0.3.0}/docs/.empty +0 -0
  114. {docler-0.2.0 → docler-0.3.0}/duties.py +0 -0
  115. {docler-0.2.0 → docler-0.3.0}/mkdocs.yml +0 -0
  116. {docler-0.2.0 → docler-0.3.0}/overrides/_dummy.txt +0 -0
  117. {docler-0.2.0 → docler-0.3.0}/src/docler/chunkers/__init__.py +0 -0
  118. {docler-0.2.0 → docler-0.3.0}/src/docler/chunkers/ai_chunker/__init__.py +0 -0
  119. {docler-0.2.0 → docler-0.3.0}/src/docler/chunkers/llamaindex_chunker/__init__.py +0 -0
  120. {docler-0.2.0 → docler-0.3.0}/src/docler/chunkers/markdown_chunker/__init__.py +0 -0
  121. {docler-0.2.0/src/docler/configs → docler-0.3.0/src/docler/converters}/__init__.py +0 -0
  122. {docler-0.2.0 → docler-0.3.0}/src/docler/converters/datalab_provider/__init__.py +0 -0
  123. {docler-0.2.0 → docler-0.3.0}/src/docler/converters/dir_converter.py +0 -0
  124. {docler-0.2.0 → docler-0.3.0}/src/docler/converters/docling_provider/__init__.py +0 -0
  125. {docler-0.2.0 → docler-0.3.0}/src/docler/converters/docling_provider/utils.py +0 -0
  126. {docler-0.2.0 → docler-0.3.0}/src/docler/converters/kreuzberg_provider/__init__.py +0 -0
  127. {docler-0.2.0 → docler-0.3.0}/src/docler/converters/llamaparse_provider/__init__.py +0 -0
  128. {docler-0.2.0 → docler-0.3.0}/src/docler/converters/llm_provider/__init__.py +0 -0
  129. {docler-0.2.0 → docler-0.3.0}/src/docler/converters/marker_provider/__init__.py +0 -0
  130. {docler-0.2.0 → docler-0.3.0}/src/docler/converters/markitdown_provider/__init__.py +0 -0
  131. {docler-0.2.0 → docler-0.3.0}/src/docler/converters/mistral_provider/__init__.py +0 -0
  132. {docler-0.2.0 → docler-0.3.0}/src/docler/converters/registry.py +0 -0
  133. {docler-0.2.0 → docler-0.3.0}/src/docler/embeddings/__init__.py +0 -0
  134. {docler-0.2.0/src/docler/converters → docler-0.3.0/src/docler/processors}/__init__.py +0 -0
  135. {docler-0.2.0 → docler-0.3.0}/src/docler/py.typed +0 -0
  136. {docler-0.2.0 → docler-0.3.0}/src/docler/resources/pdf_sample.pdf +0 -0
  137. {docler-0.2.0 → docler-0.3.0}/src/docler/streamlit_app/__init__.py +0 -0
  138. {docler-0.2.0 → docler-0.3.0}/src/docler/vector_db/__init__.py +0 -0
  139. {docler-0.2.0/src/docler/converters/azure_provider → docler-0.3.0/src/docler/vector_db/dbs}/__init__.py +0 -0
  140. {docler-0.2.0 → docler-0.3.0}/src/docler/vector_db/dbs/qdrant_db/__init__.py +0 -0
  141. {docler-0.2.0 → docler-0.3.0}/tests/__init__.py +0 -0
  142. {docler-0.2.0 → docler-0.3.0}/tests/conftest.py +0 -0
  143. {docler-0.2.0 → docler-0.3.0}/tests/resources/pdf_sample.pdf +0 -0
  144. {docler-0.2.0 → docler-0.3.0}/tests/test_chunkers.py +0 -0
  145. {docler-0.2.0 → docler-0.3.0}/tests/test_converters.py +0 -0
@@ -35,7 +35,7 @@ repos:
35
35
  additional_dependencies: [orjson, pydantic]
36
36
 
37
37
  - repo: https://github.com/charliermarsh/ruff-pre-commit
38
- rev: v0.11.0
38
+ rev: v0.11.1
39
39
  hooks:
40
40
  - id: ruff
41
41
  - id: ruff-format
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: docler
3
- Version: 0.2.0
3
+ Version: 0.3.0
4
4
  Summary: Abstractions & Tools for OCR / document processing
5
5
  Project-URL: Documentation, https://phil65.github.io/docler/
6
6
  Project-URL: Source, https://github.com/phil65/docler
@@ -46,8 +46,10 @@ Classifier: Typing :: Typed
46
46
  Requires-Python: >=3.12
47
47
  Requires-Dist: anyenv>=0.4.1
48
48
  Requires-Dist: pydantic
49
+ Requires-Dist: pydantic-settings>=2.8.1
49
50
  Requires-Dist: upathtools>=0.4.3
50
51
  Provides-Extra: all
52
+ Requires-Dist: azure-ai-documentintelligence; extra == 'all'
51
53
  Requires-Dist: chromadb; extra == 'all'
52
54
  Requires-Dist: docling; extra == 'all'
53
55
  Requires-Dist: kreuzberg; extra == 'all'
@@ -55,15 +57,23 @@ Requires-Dist: llama-index-readers-smart-pdf-loader; extra == 'all'
55
57
  Requires-Dist: llmling-agent[pydantic-ai]; extra == 'all'
56
58
  Requires-Dist: llmsherpa; extra == 'all'
57
59
  Requires-Dist: marker-pdf; extra == 'all'
58
- Requires-Dist: markitdown; extra == 'all'
60
+ Requires-Dist: markitdown[all]; extra == 'all'
59
61
  Requires-Dist: mistralai; extra == 'all'
60
62
  Requires-Dist: openai; extra == 'all'
63
+ Requires-Dist: pinecone[asyncio]; extra == 'all'
64
+ Requires-Dist: qdrant-client[fastembed]; extra == 'all'
61
65
  Requires-Dist: streamlit; extra == 'all'
62
66
  Requires-Dist: upstash-vector; extra == 'all'
67
+ Provides-Extra: azure
68
+ Requires-Dist: azure-ai-documentintelligence; extra == 'azure'
69
+ Provides-Extra: bge
70
+ Requires-Dist: flagembedding; extra == 'bge'
63
71
  Provides-Extra: chromadb
64
72
  Requires-Dist: chromadb>=0.6.3; extra == 'chromadb'
73
+ Provides-Extra: diffs
74
+ Requires-Dist: diff-match-patch; extra == 'diffs'
65
75
  Provides-Extra: docling
66
- Requires-Dist: docling; extra == 'docling'
76
+ Requires-Dist: docling[ocrmac,rapidocr,vlm]; extra == 'docling'
67
77
  Provides-Extra: kreuzberg
68
78
  Requires-Dist: kreuzberg; extra == 'kreuzberg'
69
79
  Provides-Extra: litellm
@@ -75,17 +85,23 @@ Provides-Extra: llama-parse
75
85
  Requires-Dist: llama-parse; extra == 'llama-parse'
76
86
  Provides-Extra: marker
77
87
  Requires-Dist: marker-pdf; extra == 'marker'
88
+ Provides-Extra: markitdown
89
+ Requires-Dist: markitdown[all]; extra == 'markitdown'
78
90
  Provides-Extra: mistralai
79
91
  Requires-Dist: mistralai; extra == 'mistralai'
80
92
  Provides-Extra: openai
81
93
  Requires-Dist: openai; extra == 'openai'
94
+ Provides-Extra: pinecone
95
+ Requires-Dist: pinecone[asyncio]; extra == 'pinecone'
96
+ Provides-Extra: qdrant
97
+ Requires-Dist: qdrant-client[fastembed]; extra == 'qdrant'
82
98
  Provides-Extra: smart-pdf
83
99
  Requires-Dist: llama-index-readers-smart-pdf-loader; extra == 'smart-pdf'
84
100
  Requires-Dist: llmsherpa; extra == 'smart-pdf'
85
101
  Provides-Extra: streamlit
102
+ Requires-Dist: streambricks; extra == 'streamlit'
86
103
  Requires-Dist: streamlit; extra == 'streamlit'
87
- Provides-Extra: upstash
88
- Requires-Dist: upstash-vector; extra == 'upstash'
104
+ Requires-Dist: tokonomics; extra == 'streamlit'
89
105
  Description-Content-Type: text/markdown
90
106
 
91
107
  # Docler
@@ -36,30 +36,36 @@ license = { file = "LICENSE" }
36
36
  dependencies = [
37
37
  "anyenv>=0.4.1",
38
38
  "pydantic",
39
+ "pydantic-settings>=2.8.1",
39
40
  # Only add below (Copier)
40
41
  "upathtools>=0.4.3",
41
42
  ]
42
43
 
44
+
43
45
  [project.optional-dependencies]
44
46
  marker = ["marker-pdf"]
45
47
  mistralai = ["mistralai"]
46
- docling = ["docling"]
48
+ diffs = ["diff-match-patch"]
49
+ bge = ["FlagEmbedding"]
50
+ docling = ["docling[ocrmac,vlm,rapidocr]"] # tesserocr
47
51
  # olmocr = ["olmocr", "accelerate"]
48
- # markitdown = ["markitdown[all]"]
52
+ markitdown = ["markitdown[all]"]
49
53
  kreuzberg = ["kreuzberg"]
50
54
  litellm = ["litellm", "tokonomics"]
51
- streamlit = ["streamlit"]
55
+ streamlit = ["streamlit", "streambricks", "tokonomics"]
52
56
  llama-index = ["llama-index"]
53
57
  llama-parse = ["llama-parse"]
54
58
  smart-pdf = ["llmsherpa", "llama-index-readers-smart-pdf-loader"]
55
59
  chromadb = ["chromadb>=0.6.3"]
56
- upstash = ["upstash-vector"]
57
60
  openai = ["openai"]
61
+ pinecone = ["pinecone[asyncio]"]
62
+ qdrant = ["qdrant-client[fastembed]"]
63
+ azure = ["azure-ai-documentintelligence"]
58
64
  all = [
59
65
  "marker-pdf",
60
66
  "mistralai",
61
67
  "docling",
62
- "markitdown",
68
+ "markitdown[all]",
63
69
  "kreuzberg",
64
70
  # "litellm",
65
71
  "llmling-agent[pydantic-ai]",
@@ -70,6 +76,9 @@ all = [
70
76
  "chromadb",
71
77
  "upstash-vector",
72
78
  "openai",
79
+ "pinecone[asyncio]",
80
+ "qdrant-client[fastembed]",
81
+ "azure-ai-documentintelligence",
73
82
  ]
74
83
 
75
84
 
@@ -15,7 +15,7 @@ from docler.converters.llm_provider import LLMConverter
15
15
  from docler.converters.datalab_provider import DataLabConverter
16
16
  from docler.converters.llamaparse_provider import LlamaParseConverter
17
17
 
18
- __version__ = "0.2.0"
18
+ __version__ = "0.3.0"
19
19
 
20
20
  __all__ = [
21
21
  "Conversion",
@@ -2,7 +2,6 @@
2
2
 
3
3
  from __future__ import annotations
4
4
 
5
- import logging
6
5
  import os
7
6
  import pathlib
8
7
  import subprocess
@@ -10,10 +9,12 @@ import sys
10
9
 
11
10
  import typer
12
11
 
12
+ from docler.log import get_logger
13
+
13
14
 
14
15
  cli = typer.Typer(help="Docler document converter CLI", no_args_is_help=True)
15
16
 
16
- logger = logging.getLogger(__name__)
17
+ logger = get_logger(__name__)
17
18
 
18
19
  PACKAGE_DIR = pathlib.Path(__file__).parent
19
20
 
@@ -0,0 +1 @@
1
+ """Annotators for Documents."""
@@ -0,0 +1,125 @@
1
+ """AI-powered document and chunk metadata annotation."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import asyncio
6
+ from itertools import batched
7
+ from typing import TYPE_CHECKING, ClassVar, TypeVar
8
+
9
+ from pydantic import BaseModel
10
+
11
+ from docler.annotators.base import Annotator
12
+ from docler.common_types import DEFAULT_ANNOTATOR_MODEL
13
+ from docler.configs.annotator_configs import (
14
+ DEFAULT_DOC_PROMPT_TEMPLATE,
15
+ DEFAULT_DOC_SYSTEM_PROMPT,
16
+ AIDocumentAnnotatorConfig,
17
+ )
18
+ from docler.log import get_logger
19
+
20
+
21
+ if TYPE_CHECKING:
22
+ from docler.models import ChunkedDocument
23
+
24
+ logger = get_logger(__name__)
25
+
26
+
27
+ class DefaultMetadata(BaseModel):
28
+ """Default metadata for a document or chunk."""
29
+
30
+ topics: list[str]
31
+ """Topics/categories."""
32
+
33
+ keywords: list[str]
34
+ """Keywords."""
35
+
36
+ entities: list[str]
37
+ """Main entities."""
38
+
39
+
40
+ # Type variable for generic metadata model
41
+ T = TypeVar("T", bound=BaseModel)
42
+
43
+
44
+ class AIDocumentAnnotator[TMetadata](Annotator[AIDocumentAnnotatorConfig]):
45
+ """AI-based document and chunk annotator.
46
+
47
+ Enhances documents and chunks with metadata.
48
+
49
+ Type Parameters:
50
+ T: Type of metadata model to use. Must be a Pydantic BaseModel.
51
+ """
52
+
53
+ Config = AIDocumentAnnotatorConfig
54
+ REQUIRED_PACKAGES: ClassVar = {"llmling-agent"}
55
+
56
+ def __init__(
57
+ self,
58
+ model: str | None = None,
59
+ system_prompt: str | None = None,
60
+ user_prompt: str | None = None,
61
+ metadata_model: type[TMetadata] = DefaultMetadata, # type: ignore
62
+ max_context_length: int = 1500,
63
+ batch_size: int = 5,
64
+ ):
65
+ """Initialize the AI document annotator.
66
+
67
+ Args:
68
+ model: LLM model to use for annotation
69
+ system_prompt: Optional custom prompt for annotation
70
+ user_prompt: Optional custom prompt for annotation
71
+ metadata_model: Pydantic model class for metadata structure
72
+ max_context_length: Maximum length of context for annotation
73
+ batch_size: Number of chunks to process in parallel
74
+ """
75
+ self.model = model or DEFAULT_ANNOTATOR_MODEL
76
+ self.system_prompt = system_prompt or DEFAULT_DOC_SYSTEM_PROMPT
77
+ self.user_prompt = user_prompt or DEFAULT_DOC_PROMPT_TEMPLATE
78
+ self.metadata_model = metadata_model
79
+ self.max_context_length = max_context_length
80
+ self.batch_size = batch_size
81
+
82
+ async def annotate(self, document: ChunkedDocument) -> ChunkedDocument:
83
+ """Annotate document and chunks with AI-generated metadata.
84
+
85
+ Args:
86
+ document: Chunked document to annotate
87
+
88
+ Returns:
89
+ Document with enhanced metadata
90
+ """
91
+ from llmling_agent import Agent, StructuredAgent
92
+
93
+ agent: StructuredAgent[None, TMetadata] = Agent[None]( # type: ignore
94
+ model=self.model,
95
+ system_prompt=self.system_prompt,
96
+ ).to_structured(self.metadata_model)
97
+
98
+ # Process document-level metadata if needed
99
+ if document.metadata is None:
100
+ document.metadata = {}
101
+
102
+ # Get a condensed version of the document for context
103
+ context = (
104
+ document.content[: self.max_context_length] + "..."
105
+ if self.max_context_length and len(document.content) > self.max_context_length
106
+ else document.content
107
+ )
108
+
109
+ # Process chunks in batches
110
+ for batch in batched(document.chunks, self.batch_size):
111
+ tasks = []
112
+
113
+ for chunk in batch:
114
+ prompt = self.user_prompt.format(context=context, chunk=chunk.text)
115
+ tasks.append(agent.run(prompt))
116
+
117
+ try:
118
+ results = await asyncio.gather(*tasks)
119
+ for chunk, result in zip(batch, results):
120
+ metadata = result.content.model_dump() # type: ignore
121
+ chunk.metadata |= metadata
122
+ except Exception:
123
+ logger.exception("Error annotating batch")
124
+
125
+ return document
@@ -0,0 +1,185 @@
1
+ """AI-powered image annotation."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import asyncio
6
+ import base64
7
+ from itertools import batched
8
+ from typing import TYPE_CHECKING, ClassVar
9
+
10
+ from pydantic import BaseModel
11
+ from typing_extensions import TypeVar
12
+
13
+ from docler.annotators.base import Annotator
14
+ from docler.common_types import DEFAULT_IMAGE_ANNOTATOR_MODEL
15
+ from docler.configs.annotator_configs import (
16
+ DEFAULT_IMAGE_PROMPT_TEMPLATE,
17
+ DEFAULT_IMAGE_SYSTEM_PROMPT,
18
+ AIImageAnnotatorConfig,
19
+ )
20
+ from docler.log import get_logger
21
+
22
+
23
+ if TYPE_CHECKING:
24
+ from docler.models import ChunkedDocument, Image
25
+
26
+ logger = get_logger(__name__)
27
+
28
+
29
+ class DefaultImageMetadata(BaseModel):
30
+ """Default metadata for an image."""
31
+
32
+ description: str
33
+ """Detailed description of the image."""
34
+
35
+ objects: list[str]
36
+ """Objects identified in the image."""
37
+
38
+ text_content: str | None = None
39
+ """Text visible in the image, if any."""
40
+
41
+ image_type: str
42
+ """Type of image (photo, diagram, chart, etc.)."""
43
+
44
+ colors: list[str]
45
+ """Dominant colors in the image."""
46
+
47
+
48
+ TMetadata = TypeVar("TMetadata", bound=BaseModel, default=DefaultImageMetadata)
49
+
50
+
51
+ class AIImageAnnotator[TMetadata](Annotator[AIImageAnnotatorConfig]):
52
+ """AI-based image annotator.
53
+
54
+ Analyzes images in chunks and adds descriptions and metadata.
55
+
56
+ Type Parameters:
57
+ TMetadata: Type of metadata model to use. Must be a Pydantic BaseModel.
58
+ """
59
+
60
+ Config = AIImageAnnotatorConfig
61
+
62
+ REQUIRED_PACKAGES: ClassVar = {"llmling-agent"}
63
+
64
+ def __init__(
65
+ self,
66
+ model: str | None = None,
67
+ system_prompt: str | None = None,
68
+ user_prompt: str | None = None,
69
+ metadata_model: type[TMetadata] = DefaultImageMetadata, # type: ignore
70
+ batch_size: int = 3,
71
+ ):
72
+ """Initialize the AI image annotator.
73
+
74
+ Args:
75
+ model: Vision model to use (must support images)
76
+ system_prompt: Custom prompt for image analysis
77
+ user_prompt: Custom user prompt template for image analysis
78
+ metadata_model: Pydantic model for image metadata
79
+ batch_size: Number of images to process concurrently
80
+ """
81
+ self.model = model or DEFAULT_IMAGE_ANNOTATOR_MODEL
82
+ self.system_prompt = system_prompt or DEFAULT_IMAGE_SYSTEM_PROMPT
83
+ self.user_prompt = user_prompt or DEFAULT_IMAGE_PROMPT_TEMPLATE
84
+ self.metadata_model = metadata_model
85
+ self.batch_size = batch_size
86
+
87
+ async def _process_image(self, image: Image) -> Image:
88
+ """Process a single image with the vision model.
89
+
90
+ Args:
91
+ image: Image to analyze
92
+
93
+ Returns:
94
+ Image with added description and metadata
95
+ """
96
+ from llmling_agent import Agent, ImageBase64Content, StructuredAgent
97
+
98
+ if image.description and image.metadata:
99
+ return image
100
+
101
+ if isinstance(image.content, bytes):
102
+ b64_content = base64.b64encode(image.content).decode("utf-8")
103
+ else:
104
+ b64_content = (
105
+ image.content
106
+ if not image.content.startswith("data:")
107
+ else image.content.split(",", 1)[1]
108
+ )
109
+ img_content = ImageBase64Content(data=b64_content, mime_type=image.mime_type)
110
+ agent: StructuredAgent[None, TMetadata] = Agent[None]( # type: ignore
111
+ model=self.model,
112
+ system_prompt=self.system_prompt,
113
+ ).to_structured(self.metadata_model)
114
+
115
+ try:
116
+ filename_info = f" ({image.filename})" if image.filename else ""
117
+ prompt = self.user_prompt.format(
118
+ image_id=image.id,
119
+ filename_info=filename_info,
120
+ filename=image.filename or "",
121
+ mime_type=image.mime_type,
122
+ )
123
+
124
+ result = await agent.run(prompt, img_content)
125
+ metadata = result.content.model_dump() # type: ignore
126
+ description = metadata.pop("description", None)
127
+ if description:
128
+ image.description = description
129
+ if image.metadata is None:
130
+ image.metadata = {}
131
+ image.metadata.update(metadata)
132
+
133
+ except Exception:
134
+ logger.exception("Error processing image %s", image.id)
135
+
136
+ return image
137
+
138
+ async def annotate(self, document: ChunkedDocument) -> ChunkedDocument:
139
+ """Annotate all images in the chunks with AI-generated descriptions.
140
+
141
+ Args:
142
+ document: Chunked document containing chunks with images to annotate
143
+
144
+ Returns:
145
+ Document with annotated images in chunks
146
+ """
147
+ for chunk in document.chunks:
148
+ if not chunk.images:
149
+ continue
150
+
151
+ for batch in batched(chunk.images, self.batch_size):
152
+ tasks = [self._process_image(img) for img in batch]
153
+ try:
154
+ await asyncio.gather(*tasks)
155
+ except Exception:
156
+ msg = "Error processing images in chunk %s"
157
+ logger.exception(msg, chunk.chunk_index)
158
+
159
+ return document
160
+
161
+
162
+ if __name__ == "__main__":
163
+ import asyncio
164
+
165
+ from docler.models import ChunkedDocument, Image, TextChunk
166
+
167
+ async def main():
168
+ annotator = AIImageAnnotator[DefaultImageMetadata]()
169
+ url = "https://www.a-i-stack.com/wp-content/uploads/go-x/u/93dcedb9-17f3-4aee-9b5a-3744e5e84686/image-342x342.png"
170
+ image = await Image.from_file(url)
171
+ document = ChunkedDocument(
172
+ content="test",
173
+ chunks=[
174
+ TextChunk(
175
+ text="Sample text",
176
+ source_doc_id="sample_doc_id",
177
+ images=[image],
178
+ chunk_index=0,
179
+ ),
180
+ ],
181
+ )
182
+ doc = await annotator.annotate(document)
183
+ print(doc)
184
+
185
+ asyncio.run(main())
@@ -0,0 +1,34 @@
1
+ """Base classes for text chunking implementations."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from abc import ABC, abstractmethod
6
+ from typing import TYPE_CHECKING, ClassVar
7
+
8
+ from docler.provider import BaseProvider
9
+
10
+
11
+ if TYPE_CHECKING:
12
+ from docler.configs.annotator_configs import BaseAnnotatorConfig
13
+ from docler.models import ChunkedDocument
14
+
15
+
16
+ class Annotator[TConfig](ABC, BaseProvider[TConfig]):
17
+ """Base class for chunk annotation processors."""
18
+
19
+ Config: ClassVar[type[BaseAnnotatorConfig]]
20
+
21
+ @abstractmethod
22
+ async def annotate(
23
+ self,
24
+ chunked_doc: ChunkedDocument,
25
+ ) -> ChunkedDocument:
26
+ """Annotate a chunked document with additional metadata.
27
+
28
+ Args:
29
+ chunked_doc: Chunked document containing the original content and chunks
30
+
31
+ Returns:
32
+ Chunked document with annotated chunks
33
+ """
34
+ raise NotImplementedError
@@ -0,0 +1,93 @@
1
+ """AI-based markdown chunking implementation."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import TYPE_CHECKING, Any, ClassVar
6
+
7
+ from docler.chunkers.ai_chunker.models import Chunk, Chunks
8
+ from docler.chunkers.ai_chunker.utils import add_line_numbers, create_text_chunk
9
+ from docler.chunkers.base import TextChunker
10
+ from docler.common_types import DEFAULT_CHUNKER_MODEL
11
+ from docler.configs.chunker_configs import (
12
+ DEFAULT_CHUNKER_SYSTEM_PROMPT,
13
+ DEFAULT_CHUNKER_USER_TEMPLATE,
14
+ )
15
+ from docler.models import TextChunk
16
+
17
+
18
+ if TYPE_CHECKING:
19
+ from docler.models import Document
20
+
21
+
22
+ class AIChunker(TextChunker):
23
+ """LLM-based document chunker."""
24
+
25
+ REQUIRED_PACKAGES: ClassVar = {"llmling-agent"}
26
+
27
+ def __init__(
28
+ self,
29
+ model: str | None = None,
30
+ user_prompt: str | None = None,
31
+ system_prompt: str | None = None,
32
+ ):
33
+ """Initialize the AI chunker.
34
+
35
+ Args:
36
+ model: LLM model to use
37
+ provider: LLM provider to use
38
+ user_prompt: User prompt to use
39
+ system_prompt: System prompt to use
40
+ """
41
+ self.model = model or DEFAULT_CHUNKER_MODEL
42
+ self.user_prompt = user_prompt or DEFAULT_CHUNKER_USER_TEMPLATE
43
+ self.system_prompt = system_prompt or DEFAULT_CHUNKER_SYSTEM_PROMPT
44
+
45
+ async def _get_chunks(self, text: str) -> Chunks:
46
+ """Get chunk definitions from LLM."""
47
+ from llmling_agent import Agent
48
+
49
+ numbered_text = add_line_numbers(text)
50
+ # agent: llmling_agent.StructuredAgent[None, Chunks] = llmling_agent.Agent[None](
51
+ # model=self.model,
52
+ # system_prompt=self.system_prompt,
53
+ # ).to_structured(Chunks)
54
+ # prompt = CHUNKING_PROMPT.format(numbered_text=numbered_text)
55
+ # response = await agent.run(prompt)
56
+ agent: Agent[None] = Agent[None](
57
+ model=self.model, system_prompt=self.system_prompt
58
+ )
59
+ prompt = self.user_prompt.format(numbered_text=numbered_text)
60
+ chunks = await agent.talk.extract_multiple(
61
+ text,
62
+ Chunk,
63
+ prompt=prompt,
64
+ mode="structured", # tool_calls
65
+ )
66
+ return Chunks(chunks=chunks)
67
+
68
+ async def split(
69
+ self,
70
+ doc: Document,
71
+ extra_metadata: dict[str, Any] | None = None,
72
+ ) -> list[TextChunk]:
73
+ """Split document into chunks using LLM analysis."""
74
+ chunks = await self._get_chunks(doc.content)
75
+ return [
76
+ create_text_chunk(doc, chunk, i, extra_metadata)
77
+ for i, chunk in enumerate(chunks.chunks)
78
+ ]
79
+
80
+
81
+ if __name__ == "__main__":
82
+ import asyncio
83
+
84
+ from docler.models import Document
85
+
86
+ async def main():
87
+ # Example usage
88
+ doc = Document(source_path="example.txt", content=DEFAULT_CHUNKER_SYSTEM_PROMPT)
89
+ chunker = AIChunker()
90
+ chunks = await chunker.split(doc)
91
+ print(chunks)
92
+
93
+ asyncio.run(main())
@@ -0,0 +1,30 @@
1
+ """Models for AI chunker."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from pydantic import BaseModel, ConfigDict
6
+
7
+
8
+ class Chunk(BaseModel):
9
+ """A chunk of text with semantic metadata."""
10
+
11
+ start_row: int
12
+ """Start line number (1-based)"""
13
+
14
+ end_row: int
15
+ """End line number (1-based)"""
16
+
17
+ keywords: list[str]
18
+ """Key terms and concepts in this chunk"""
19
+
20
+ references: list[int]
21
+ """Line numbers that this chunk references or depends on"""
22
+
23
+ model_config = ConfigDict(use_attribute_docstrings=True)
24
+
25
+
26
+ class Chunks(BaseModel):
27
+ """Collection of chunks with their metadata."""
28
+
29
+ chunks: list[Chunk]
30
+ """A list of chunks to extract from the document."""
@@ -0,0 +1,40 @@
1
+ """AI-based markdown chunking implementation."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import TYPE_CHECKING, Any
6
+
7
+ from docler.models import TextChunk
8
+
9
+
10
+ if TYPE_CHECKING:
11
+ from docler.chunkers.ai_chunker.models import Chunk
12
+ from docler.models import Document
13
+
14
+
15
+ def add_line_numbers(text: str) -> str:
16
+ """Add line numbers to text."""
17
+ lines = text.splitlines()
18
+ return "\n".join(f"{i + 1:3d} | {line}" for i, line in enumerate(lines))
19
+
20
+
21
+ def create_text_chunk(
22
+ doc: Document,
23
+ chunk: Chunk,
24
+ chunk_idx: int,
25
+ extra_metadata: dict[str, Any] | None = None,
26
+ ) -> TextChunk:
27
+ """Create a TextChunk from chunk definition."""
28
+ lines = doc.content.splitlines()
29
+ chunk_lines = lines[chunk.start_row - 1 : chunk.end_row]
30
+ chunk_text = "\n".join(chunk_lines)
31
+ base = extra_metadata or {}
32
+ metadata = {**base, "keywords": chunk.keywords, "references": chunk.references}
33
+ chunk_images = [i for i in doc.images if i.filename and i.filename in chunk_text]
34
+ return TextChunk(
35
+ text=chunk_text,
36
+ source_doc_id=doc.source_path or "",
37
+ chunk_index=chunk_idx,
38
+ images=chunk_images,
39
+ metadata=metadata,
40
+ )