docler 2.0.0__tar.gz → 2.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (96) hide show
  1. {docler-2.0.0 → docler-2.1.0}/PKG-INFO +3 -3
  2. {docler-2.0.0 → docler-2.1.0}/pyproject.toml +4 -3
  3. {docler-2.0.0 → docler-2.1.0}/src/docler/__main__.py +133 -133
  4. {docler-2.0.0 → docler-2.1.0}/src/docler/annotators/__init__.py +1 -1
  5. {docler-2.0.0 → docler-2.1.0}/src/docler/annotators/ai_document_annotator.py +3 -3
  6. {docler-2.0.0 → docler-2.1.0}/src/docler/annotators/ai_image_annotator.py +11 -5
  7. {docler-2.0.0 → docler-2.1.0}/src/docler/annotators/base.py +36 -36
  8. {docler-2.0.0 → docler-2.1.0}/src/docler/chunkers/ai_chunker/chunker.py +4 -9
  9. {docler-2.0.0 → docler-2.1.0}/src/docler/chunkers/llamaindex_chunker/chunker.py +1 -1
  10. {docler-2.0.0 → docler-2.1.0}/src/docler/chunkers/markdown_chunker/chunker.py +1 -1
  11. {docler-2.0.0 → docler-2.1.0}/src/docler/chunkers/token_chunker/chunker.py +1 -1
  12. {docler-2.0.0 → docler-2.1.0}/src/docler/common_types.py +49 -49
  13. {docler-2.0.0 → docler-2.1.0}/src/docler/converters/aggregated_converter.py +2 -6
  14. {docler-2.0.0 → docler-2.1.0}/src/docler/converters/azure_provider/__init__.py +5 -5
  15. {docler-2.0.0 → docler-2.1.0}/src/docler/converters/azure_provider/provider.py +2 -2
  16. {docler-2.0.0 → docler-2.1.0}/src/docler/converters/base.py +56 -15
  17. {docler-2.0.0 → docler-2.1.0}/src/docler/converters/datalab_provider/__init__.py +5 -5
  18. {docler-2.0.0 → docler-2.1.0}/src/docler/converters/datalab_provider/provider.py +2 -6
  19. {docler-2.0.0 → docler-2.1.0}/src/docler/converters/dir_converter.py +179 -179
  20. {docler-2.0.0 → docler-2.1.0}/src/docler/converters/docling_provider/__init__.py +5 -5
  21. {docler-2.0.0 → docler-2.1.0}/src/docler/converters/docling_provider/provider.py +2 -2
  22. {docler-2.0.0 → docler-2.1.0}/src/docler/converters/docling_provider/utils.py +82 -82
  23. {docler-2.0.0 → docler-2.1.0}/src/docler/converters/docling_remote_provider/provider.py +2 -6
  24. {docler-2.0.0 → docler-2.1.0}/src/docler/converters/exceptions.py +9 -9
  25. {docler-2.0.0 → docler-2.1.0}/src/docler/converters/llamaparse_provider/__init__.py +5 -5
  26. {docler-2.0.0 → docler-2.1.0}/src/docler/converters/llamaparse_provider/provider.py +2 -2
  27. {docler-2.0.0 → docler-2.1.0}/src/docler/converters/llm_provider/__init__.py +5 -5
  28. {docler-2.0.0 → docler-2.1.0}/src/docler/converters/llm_provider/provider.py +8 -7
  29. {docler-2.0.0 → docler-2.1.0}/src/docler/converters/marker_provider/__init__.py +5 -5
  30. {docler-2.0.0 → docler-2.1.0}/src/docler/converters/marker_provider/provider.py +1 -1
  31. {docler-2.0.0 → docler-2.1.0}/src/docler/converters/markitdown_provider/__init__.py +5 -5
  32. {docler-2.0.0 → docler-2.1.0}/src/docler/converters/markitdown_provider/provider.py +1 -1
  33. {docler-2.0.0 → docler-2.1.0}/src/docler/converters/mistral_provider/__init__.py +5 -5
  34. {docler-2.0.0 → docler-2.1.0}/src/docler/converters/mistral_provider/provider.py +1 -1
  35. {docler-2.0.0 → docler-2.1.0}/src/docler/converters/upstage_provider/__init__.py +5 -5
  36. {docler-2.0.0 → docler-2.1.0}/src/docler/converters/upstage_provider/provider.py +2 -2
  37. {docler-2.0.0 → docler-2.1.0}/src/docler/diffs.py +90 -90
  38. {docler-2.0.0 → docler-2.1.0}/src/docler/mime_types.py +174 -174
  39. {docler-2.0.0 → docler-2.1.0}/src/docler/process_runner.py +276 -276
  40. {docler-2.0.0 → docler-2.1.0}/src/docler/processors/ai_processor.py +8 -10
  41. {docler-2.0.0 → docler-2.1.0}/src/docler/processors/base.py +35 -35
  42. {docler-2.0.0 → docler-2.1.0}/src/docler/provider.py +66 -99
  43. {docler-2.0.0 → docler-2.1.0}/src/docler_api/__init__.py +3 -3
  44. {docler-2.0.0 → docler-2.1.0}/src/docler_api/main.py +2 -2
  45. {docler-2.0.0 → docler-2.1.0}/src/docler_api/routes.py +35 -112
  46. {docler-2.0.0/src/docler/configs → docler-2.1.0/src/docler_config}/__init__.py +91 -91
  47. {docler-2.0.0/src/docler/configs → docler-2.1.0/src/docler_config}/annotator_configs.py +117 -115
  48. {docler-2.0.0/src/docler/configs → docler-2.1.0/src/docler_config}/chunker_configs.py +146 -145
  49. {docler-2.0.0/src/docler/configs → docler-2.1.0/src/docler_config}/converter_configs.py +5 -6
  50. {docler-2.0.0/src/docler/configs → docler-2.1.0/src/docler_config}/processor_configs.py +79 -79
  51. docler-2.1.0/src/docler_config/provider.py +44 -0
  52. {docler-2.0.0 → docler-2.1.0}/src/docler_streamlit/__init__.py +6 -6
  53. {docler-2.0.0 → docler-2.1.0}/src/docler_streamlit/ocr_app.py +56 -56
  54. {docler-2.0.0 → docler-2.1.0}/LICENSE +0 -0
  55. {docler-2.0.0 → docler-2.1.0}/README.md +0 -0
  56. {docler-2.0.0 → docler-2.1.0}/src/docler/__init__.py +0 -0
  57. {docler-2.0.0 → docler-2.1.0}/src/docler/chunkers/__init__.py +0 -0
  58. {docler-2.0.0 → docler-2.1.0}/src/docler/chunkers/ai_chunker/__init__.py +0 -0
  59. {docler-2.0.0 → docler-2.1.0}/src/docler/chunkers/ai_chunker/models.py +0 -0
  60. {docler-2.0.0 → docler-2.1.0}/src/docler/chunkers/ai_chunker/utils.py +0 -0
  61. {docler-2.0.0 → docler-2.1.0}/src/docler/chunkers/base.py +0 -0
  62. {docler-2.0.0 → docler-2.1.0}/src/docler/chunkers/llamaindex_chunker/__init__.py +0 -0
  63. {docler-2.0.0 → docler-2.1.0}/src/docler/chunkers/markdown_chunker/__init__.py +0 -0
  64. {docler-2.0.0 → docler-2.1.0}/src/docler/chunkers/markdown_chunker/utils.py +0 -0
  65. {docler-2.0.0 → docler-2.1.0}/src/docler/chunkers/token_chunker/__init__.py +0 -0
  66. {docler-2.0.0 → docler-2.1.0}/src/docler/converters/__init__.py +0 -0
  67. {docler-2.0.0 → docler-2.1.0}/src/docler/converters/azure_provider/utils.py +0 -0
  68. {docler-2.0.0 → docler-2.1.0}/src/docler/converters/datalab_provider/utils.py +0 -0
  69. {docler-2.0.0 → docler-2.1.0}/src/docler/converters/docling_remote_provider/__init__.py +0 -0
  70. {docler-2.0.0 → docler-2.1.0}/src/docler/converters/docling_remote_provider/utils.py +0 -0
  71. {docler-2.0.0 → docler-2.1.0}/src/docler/converters/llamaparse_provider/utils.py +0 -0
  72. {docler-2.0.0 → docler-2.1.0}/src/docler/converters/mistral_provider/utils.py +0 -0
  73. {docler-2.0.0 → docler-2.1.0}/src/docler/converters/registry.py +0 -0
  74. {docler-2.0.0 → docler-2.1.0}/src/docler/log.py +0 -0
  75. {docler-2.0.0 → docler-2.1.0}/src/docler/models.py +0 -0
  76. {docler-2.0.0 → docler-2.1.0}/src/docler/pdf_utils.py +0 -0
  77. {docler-2.0.0 → docler-2.1.0}/src/docler/processors/__init__.py +0 -0
  78. {docler-2.0.0 → docler-2.1.0}/src/docler/py.typed +0 -0
  79. {docler-2.0.0 → docler-2.1.0}/src/docler/resources/pdf_sample.pdf +0 -0
  80. {docler-2.0.0 → docler-2.1.0}/src/docler/resources/samplepptx.pptx +0 -0
  81. {docler-2.0.0 → docler-2.1.0}/src/docler/resources/test.docx +0 -0
  82. {docler-2.0.0 → docler-2.1.0}/src/docler/resources/test.jpg +0 -0
  83. {docler-2.0.0 → docler-2.1.0}/src/docler/resources/test.pptx +0 -0
  84. {docler-2.0.0 → docler-2.1.0}/src/docler/resources/test.xls +0 -0
  85. {docler-2.0.0 → docler-2.1.0}/src/docler/resources/test.xlsx +0 -0
  86. {docler-2.0.0 → docler-2.1.0}/src/docler/utils.py +0 -0
  87. {docler-2.0.0 → docler-2.1.0}/src/docler_api/README.md +0 -0
  88. {docler-2.0.0 → docler-2.1.0}/src/docler_api/py.typed +0 -0
  89. {docler-2.0.0 → docler-2.1.0}/src/docler_mcp/__init__.py +0 -0
  90. {docler-2.0.0 → docler-2.1.0}/src/docler_mcp/py.typed +0 -0
  91. {docler-2.0.0 → docler-2.1.0}/src/docler_mcp/server.py +0 -0
  92. {docler-2.0.0 → docler-2.1.0}/src/docler_streamlit/chunk_app.py +0 -0
  93. {docler-2.0.0 → docler-2.1.0}/src/docler_streamlit/chunkers.py +0 -0
  94. {docler-2.0.0 → docler-2.1.0}/src/docler_streamlit/converters.py +0 -0
  95. {docler-2.0.0 → docler-2.1.0}/src/docler_streamlit/py.typed +0 -0
  96. {docler-2.0.0 → docler-2.1.0}/src/docler_streamlit/utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: docler
3
- Version: 2.0.0
3
+ Version: 2.1.0
4
4
  Summary: Abstractions & Tools for OCR / document processing
5
5
  Keywords:
6
6
  Author: Philipp Temminghoff
@@ -30,12 +30,12 @@ Requires-Dist: pytest-docker>=3.2.1
30
30
  Requires-Dist: schemez>=0.0.1
31
31
  Requires-Dist: upathtools>=0.4.3
32
32
  Requires-Dist: docler[marker,azure,diffs,docling,llama-parse,marker,markitdown,mcp,mistralai,openai,server] ; extra == 'all'
33
- Requires-Dist: llmling-agent[default] ; extra == 'all'
33
+ Requires-Dist: agentpool[default] ; extra == 'all'
34
34
  Requires-Dist: azure-ai-documentintelligence ; extra == 'azure'
35
35
  Requires-Dist: diff-match-patch ; extra == 'diffs'
36
36
  Requires-Dist: docling[ocrmac,vlm,rapidocr] ; python_full_version < '3.14' and extra == 'docling'
37
37
  Requires-Dist: docler[mistralai,diffs,llama-parse,openai,azure] ; extra == 'light'
38
- Requires-Dist: llmling-agent[default] ; extra == 'light'
38
+ Requires-Dist: agentpool[default] ; extra == 'light'
39
39
  Requires-Dist: llama-parse ; python_full_version < '3.14' and extra == 'llama-parse'
40
40
  Requires-Dist: marker-pdf ; python_full_version < '3.14' and extra == 'marker'
41
41
  Requires-Dist: markitdown[all] ; python_full_version < '3.14' and extra == 'markitdown'
@@ -2,7 +2,7 @@
2
2
 
3
3
  [project]
4
4
  name = "docler"
5
- version = "2.0.0"
5
+ version = "2.1.0"
6
6
  description = "Abstractions & Tools for OCR / document processing"
7
7
  readme = "README.md"
8
8
  requires-python = ">=3.13"
@@ -54,7 +54,7 @@ docler = "docler.__main__:cli"
54
54
  [project.optional-dependencies]
55
55
  all = [
56
56
  "docler[marker,azure,diffs,docling,llama-parse,marker,markitdown,mcp,mistralai,openai,server]",
57
- "llmling-agent[default]"
57
+ "agentpool[default]"
58
58
  ]
59
59
  azure = ["azure-ai-documentintelligence"]
60
60
  diffs = ["diff-match-patch"]
@@ -63,7 +63,7 @@ docling = [
63
63
  ] # tesserocr
64
64
  light = [
65
65
  "docler[mistralai,diffs,llama-parse,openai,azure]",
66
- "llmling-agent[default]"
66
+ "agentpool[default]"
67
67
  ]
68
68
  llama-parse = ["llama-parse; python_version < '3.14'"]
69
69
  marker = ["marker-pdf; python_version < '3.14'"]
@@ -316,6 +316,7 @@ default-groups = ["dev", "lint", "docs"]
316
316
  [tool.uv.build-backend]
317
317
  module-name = [
318
318
  "docler",
319
+ "docler_config",
319
320
  "docler_streamlit",
320
321
  "docler_api",
321
322
  "docler_mcp",
@@ -1,133 +1,133 @@
1
- """Command line interface for Docler document converter."""
2
-
3
- from __future__ import annotations
4
-
5
- from enum import Enum
6
- import os
7
- import pathlib
8
- import subprocess
9
- import sys
10
-
11
- import typer
12
-
13
- from docler.log import get_logger
14
-
15
-
16
- cli = typer.Typer(help="Docler document converter CLI", no_args_is_help=True)
17
-
18
- logger = get_logger(__name__)
19
-
20
- PACKAGE_DIR = pathlib.Path(__file__).parent.parent / "docler_streamlit"
21
-
22
-
23
- class TransportType(str, Enum):
24
- """Transport protocol types for MCP server."""
25
-
26
- STDIO = "stdio"
27
- SSE = "sse"
28
- STREAMABLE_HTTP = "streamable-http"
29
-
30
-
31
- @cli.command()
32
- def flow() -> None:
33
- """Start the Streamlit web interface."""
34
- app_path = str(PACKAGE_DIR / "app.py")
35
- try:
36
- cmd = [sys.executable, "-m", "streamlit", "run", app_path]
37
- subprocess.run(cmd, env=os.environ.copy(), check=True)
38
- except subprocess.CalledProcessError as e:
39
- # msg = f"Failed to start Streamlit: {e}"
40
- raise typer.Exit(1) from e
41
- except KeyboardInterrupt:
42
- logger.info("Shutting down...")
43
-
44
-
45
- @cli.command("chunk_ui")
46
- def chunk_ui() -> None:
47
- """Start the Streamlit web interface."""
48
- app_path = str(PACKAGE_DIR / "chunk_app.py")
49
- try:
50
- cmd = [sys.executable, "-m", "streamlit", "run", app_path]
51
- subprocess.run(cmd, env=os.environ.copy(), check=True)
52
- except subprocess.CalledProcessError as e:
53
- # msg = f"Failed to start Streamlit: {e}"
54
- raise typer.Exit(1) from e
55
- except KeyboardInterrupt:
56
- logger.info("Shutting down...")
57
-
58
-
59
- @cli.command("ocr_ui")
60
- def ocr_ui() -> None:
61
- """Start the Streamlit web interface."""
62
- app_path = str(PACKAGE_DIR / "ocr_app.py")
63
- try:
64
- cmd = [sys.executable, "-m", "streamlit", "run", app_path]
65
- subprocess.run(cmd, env=os.environ.copy(), check=True)
66
- except subprocess.CalledProcessError as e:
67
- # msg = f"Failed to start Streamlit: {e}"
68
- raise typer.Exit(1) from e
69
- except KeyboardInterrupt:
70
- logger.info("Shutting down...")
71
-
72
-
73
- @cli.command()
74
- def api(
75
- host: str = typer.Option("0.0.0.0", help="Host to bind to"),
76
- port: int = typer.Option(8000, help="Port to bind to"),
77
- log_level: str = typer.Option("info", help="Log level"),
78
- reload: bool = typer.Option(False, help="Enable auto-reload on file changes"),
79
- ) -> None:
80
- """Start the Docler API server."""
81
- try:
82
- # Lazy import to avoid unnecessary dependencies if not using the API
83
- import uvicorn
84
-
85
- from docler_api.main import app
86
-
87
- uvicorn.run(
88
- app,
89
- host=host,
90
- port=port,
91
- log_level=log_level,
92
- reload=reload,
93
- )
94
- except ImportError as e:
95
- msg = f"Failed to import API components: {e}. Is the 'server' extra installed?"
96
- logger.exception(msg)
97
- raise typer.Exit(1) from e
98
- except KeyboardInterrupt:
99
- logger.info("Shutting down...")
100
-
101
-
102
- @cli.command()
103
- def mcp(
104
- transport: TransportType = typer.Option( # noqa: B008
105
- TransportType.STDIO, help="Transport protocol to use"
106
- ),
107
- host: str = typer.Option("0.0.0.0", help="Host to bind the server to"),
108
- port: int = typer.Option(8000, help="Port to bind the server to"),
109
- ) -> None:
110
- """Start the MCP server."""
111
- try:
112
- # Lazy import to avoid unnecessary dependencies if not using MCP
113
- from docler_mcp.server import mcp as mcp_server
114
-
115
- if transport == TransportType.STDIO:
116
- mcp_server.run(transport=transport.value)
117
- else:
118
- mcp_server.run(transport=transport.value, host=host, port=port)
119
- except ImportError as e:
120
- msg = f"Failed to import MCP components: {e}. Is the 'mcp' extra installed?"
121
- logger.exception(msg)
122
- raise typer.Exit(1) from e
123
- except KeyboardInterrupt:
124
- logger.info("Shutting down...")
125
-
126
-
127
- def main() -> None:
128
- """Main entry point."""
129
- cli()
130
-
131
-
132
- if __name__ == "__main__":
133
- main()
1
+ """Command line interface for Docler document converter."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from enum import Enum
6
+ import os
7
+ import pathlib
8
+ import subprocess
9
+ import sys
10
+
11
+ import typer
12
+
13
+ from docler.log import get_logger
14
+
15
+
16
+ cli = typer.Typer(help="Docler document converter CLI", no_args_is_help=True)
17
+
18
+ logger = get_logger(__name__)
19
+
20
+ PACKAGE_DIR = pathlib.Path(__file__).parent.parent / "docler_streamlit"
21
+
22
+
23
+ class TransportType(str, Enum):
24
+ """Transport protocol types for MCP server."""
25
+
26
+ STDIO = "stdio"
27
+ SSE = "sse"
28
+ STREAMABLE_HTTP = "streamable-http"
29
+
30
+
31
+ @cli.command()
32
+ def flow() -> None:
33
+ """Start the Streamlit web interface."""
34
+ app_path = str(PACKAGE_DIR / "app.py")
35
+ try:
36
+ cmd = [sys.executable, "-m", "streamlit", "run", app_path]
37
+ subprocess.run(cmd, env=os.environ.copy(), check=True)
38
+ except subprocess.CalledProcessError as e:
39
+ # msg = f"Failed to start Streamlit: {e}"
40
+ raise typer.Exit(1) from e
41
+ except KeyboardInterrupt:
42
+ logger.info("Shutting down...")
43
+
44
+
45
+ @cli.command("chunk_ui")
46
+ def chunk_ui() -> None:
47
+ """Start the Streamlit web interface."""
48
+ app_path = str(PACKAGE_DIR / "chunk_app.py")
49
+ try:
50
+ cmd = [sys.executable, "-m", "streamlit", "run", app_path]
51
+ subprocess.run(cmd, env=os.environ.copy(), check=True)
52
+ except subprocess.CalledProcessError as e:
53
+ # msg = f"Failed to start Streamlit: {e}"
54
+ raise typer.Exit(1) from e
55
+ except KeyboardInterrupt:
56
+ logger.info("Shutting down...")
57
+
58
+
59
+ @cli.command("ocr_ui")
60
+ def ocr_ui() -> None:
61
+ """Start the Streamlit web interface."""
62
+ app_path = str(PACKAGE_DIR / "ocr_app.py")
63
+ try:
64
+ cmd = [sys.executable, "-m", "streamlit", "run", app_path]
65
+ subprocess.run(cmd, env=os.environ.copy(), check=True)
66
+ except subprocess.CalledProcessError as e:
67
+ # msg = f"Failed to start Streamlit: {e}"
68
+ raise typer.Exit(1) from e
69
+ except KeyboardInterrupt:
70
+ logger.info("Shutting down...")
71
+
72
+
73
+ @cli.command()
74
+ def api(
75
+ host: str = typer.Option("0.0.0.0", help="Host to bind to"),
76
+ port: int = typer.Option(8000, help="Port to bind to"),
77
+ log_level: str = typer.Option("info", help="Log level"),
78
+ reload: bool = typer.Option(False, help="Enable auto-reload on file changes"),
79
+ ) -> None:
80
+ """Start the Docler API server."""
81
+ try:
82
+ # Lazy import to avoid unnecessary dependencies if not using the API
83
+ import uvicorn
84
+
85
+ from docler_api.main import app
86
+
87
+ uvicorn.run(
88
+ app,
89
+ host=host,
90
+ port=port,
91
+ log_level=log_level,
92
+ reload=reload,
93
+ )
94
+ except ImportError as e:
95
+ msg = f"Failed to import API components: {e}. Is the 'server' extra installed?"
96
+ logger.exception(msg)
97
+ raise typer.Exit(1) from e
98
+ except KeyboardInterrupt:
99
+ logger.info("Shutting down...")
100
+
101
+
102
+ @cli.command()
103
+ def mcp(
104
+ transport: TransportType = typer.Option( # noqa: B008
105
+ TransportType.STDIO, help="Transport protocol to use"
106
+ ),
107
+ host: str = typer.Option("0.0.0.0", help="Host to bind the server to"),
108
+ port: int = typer.Option(8000, help="Port to bind the server to"),
109
+ ) -> None:
110
+ """Start the MCP server."""
111
+ try:
112
+ # Lazy import to avoid unnecessary dependencies if not using MCP
113
+ from docler_mcp.server import mcp as mcp_server
114
+
115
+ if transport == TransportType.STDIO:
116
+ mcp_server.run(transport=transport.value)
117
+ else:
118
+ mcp_server.run(transport=transport.value, host=host, port=port)
119
+ except ImportError as e:
120
+ msg = f"Failed to import MCP components: {e}. Is the 'mcp' extra installed?"
121
+ logger.exception(msg)
122
+ raise typer.Exit(1) from e
123
+ except KeyboardInterrupt:
124
+ logger.info("Shutting down...")
125
+
126
+
127
+ def main() -> None:
128
+ """Main entry point."""
129
+ cli()
130
+
131
+
132
+ if __name__ == "__main__":
133
+ main()
@@ -1 +1 @@
1
- """Annotators for Documents."""
1
+ """Annotators for Documents."""
@@ -10,7 +10,7 @@ from schemez import Schema
10
10
 
11
11
  from docler.annotators.base import Annotator
12
12
  from docler.common_types import DEFAULT_ANNOTATOR_MODEL
13
- from docler.configs.annotator_configs import (
13
+ from docler_config.annotator_configs import (
14
14
  DEFAULT_DOC_PROMPT_TEMPLATE,
15
15
  DEFAULT_DOC_SYSTEM_PROMPT,
16
16
  AIDocumentAnnotatorConfig,
@@ -46,7 +46,7 @@ class AIDocumentAnnotator[TMetadata: Schema = DefaultMetadata](
46
46
  """
47
47
 
48
48
  Config = AIDocumentAnnotatorConfig
49
- REQUIRED_PACKAGES: ClassVar = {"llmling-agent"}
49
+ REQUIRED_PACKAGES: ClassVar = {"agentpool"}
50
50
 
51
51
  def __init__(
52
52
  self,
@@ -84,7 +84,7 @@ class AIDocumentAnnotator[TMetadata: Schema = DefaultMetadata](
84
84
  Returns:
85
85
  Document with enhanced metadata
86
86
  """
87
- from llmling_agent import Agent
87
+ from agentpool import Agent
88
88
 
89
89
  agent = Agent(
90
90
  model=self.model,
@@ -10,7 +10,7 @@ from schemez import Schema
10
10
 
11
11
  from docler.annotators.base import Annotator
12
12
  from docler.common_types import DEFAULT_IMAGE_ANNOTATOR_MODEL
13
- from docler.configs.annotator_configs import (
13
+ from docler_config.annotator_configs import (
14
14
  DEFAULT_IMAGE_PROMPT_TEMPLATE,
15
15
  DEFAULT_IMAGE_SYSTEM_PROMPT,
16
16
  AIImageAnnotatorConfig,
@@ -53,7 +53,7 @@ class AIImageAnnotator[TMetadata: Schema = DefaultImageMetadata](Annotator[AIIma
53
53
 
54
54
  Config = AIImageAnnotatorConfig
55
55
 
56
- REQUIRED_PACKAGES: ClassVar = {"llmling-agent"}
56
+ REQUIRED_PACKAGES: ClassVar = {"agentpool"}
57
57
 
58
58
  def __init__(
59
59
  self,
@@ -88,13 +88,19 @@ class AIImageAnnotator[TMetadata: Schema = DefaultImageMetadata](Annotator[AIIma
88
88
  Returns:
89
89
  Image with added description and metadata
90
90
  """
91
- from llmling_agent import Agent, ImageBase64Content
91
+ import base64
92
+
93
+ from agentpool import Agent
94
+ from pydantic_ai import BinaryImage
92
95
 
93
96
  if image.description and image.metadata:
94
97
  return image
95
98
 
96
- b64_content = image.to_base64()
97
- img_content = ImageBase64Content(data=b64_content, mime_type=image.mime_type)
99
+ if isinstance(image.content, bytes):
100
+ data = image.content
101
+ else:
102
+ data = base64.b64decode(image.content)
103
+ img_content = BinaryImage(data=data, media_type=image.mime_type)
98
104
  agent = Agent(
99
105
  model=self.model,
100
106
  system_prompt=self.system_prompt,
@@ -1,36 +1,36 @@
1
- """Base classes for text chunking implementations."""
2
-
3
- from __future__ import annotations
4
-
5
- from abc import ABC, abstractmethod
6
- from typing import TYPE_CHECKING, ClassVar
7
-
8
- from pydantic import BaseModel
9
-
10
- from docler.provider import BaseProvider
11
-
12
-
13
- if TYPE_CHECKING:
14
- from docler.configs.annotator_configs import BaseAnnotatorConfig
15
- from docler.models import ChunkedDocument
16
-
17
-
18
- class Annotator[TConfig: BaseModel](ABC, BaseProvider[TConfig]):
19
- """Base class for chunk annotation processors."""
20
-
21
- Config: ClassVar[type[BaseAnnotatorConfig]]
22
-
23
- @abstractmethod
24
- async def annotate(
25
- self,
26
- chunked_doc: ChunkedDocument,
27
- ) -> ChunkedDocument:
28
- """Annotate a chunked document with additional metadata.
29
-
30
- Args:
31
- chunked_doc: Chunked document containing the original content and chunks
32
-
33
- Returns:
34
- Chunked document with annotated chunks
35
- """
36
- raise NotImplementedError
1
+ """Base classes for text chunking implementations."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from abc import ABC, abstractmethod
6
+ from typing import TYPE_CHECKING, ClassVar
7
+
8
+ from pydantic import BaseModel
9
+
10
+ from docler.provider import BaseProvider
11
+
12
+
13
+ if TYPE_CHECKING:
14
+ from docler.models import ChunkedDocument
15
+ from docler_config.annotator_configs import BaseAnnotatorConfig
16
+
17
+
18
+ class Annotator[TConfig: BaseModel](ABC, BaseProvider[TConfig]):
19
+ """Base class for chunk annotation processors."""
20
+
21
+ Config: ClassVar[type[BaseAnnotatorConfig]]
22
+
23
+ @abstractmethod
24
+ async def annotate(
25
+ self,
26
+ chunked_doc: ChunkedDocument,
27
+ ) -> ChunkedDocument:
28
+ """Annotate a chunked document with additional metadata.
29
+
30
+ Args:
31
+ chunked_doc: Chunked document containing the original content and chunks
32
+
33
+ Returns:
34
+ Chunked document with annotated chunks
35
+ """
36
+ raise NotImplementedError
@@ -8,7 +8,7 @@ from docler.chunkers.ai_chunker.models import Chunk, Chunks
8
8
  from docler.chunkers.ai_chunker.utils import add_line_numbers, create_text_chunk
9
9
  from docler.chunkers.base import TextChunker
10
10
  from docler.common_types import DEFAULT_CHUNKER_MODEL
11
- from docler.configs.chunker_configs import (
11
+ from docler_config.chunker_configs import (
12
12
  DEFAULT_CHUNKER_SYSTEM_PROMPT,
13
13
  DEFAULT_CHUNKER_USER_TEMPLATE,
14
14
  AiChunkerConfig,
@@ -23,7 +23,7 @@ class AIChunker(TextChunker[AiChunkerConfig]):
23
23
  """LLM-based document chunker."""
24
24
 
25
25
  NAME = "ai"
26
- REQUIRED_PACKAGES: ClassVar = {"llmling-agent"}
26
+ REQUIRED_PACKAGES: ClassVar = {"agentpool"}
27
27
  Config = AiChunkerConfig
28
28
 
29
29
  def __init__(
@@ -46,7 +46,7 @@ class AIChunker(TextChunker[AiChunkerConfig]):
46
46
 
47
47
  async def _get_chunks(self, text: str) -> Chunks:
48
48
  """Get chunk definitions from LLM."""
49
- from llmling_agent import Agent
49
+ from agentpool import Agent
50
50
 
51
51
  numbered_text = add_line_numbers(text)
52
52
  # agent: llmling_agent.StructuredAgent[None, Chunks] = llmling_agent.Agent[None](
@@ -57,12 +57,7 @@ class AIChunker(TextChunker[AiChunkerConfig]):
57
57
  # response = await agent.run(prompt)
58
58
  agent: Agent[None] = Agent[None](model=self.model, system_prompt=self.system_prompt)
59
59
  prompt = self.user_prompt.format(numbered_text=numbered_text)
60
- chunks = await agent.talk.extract_multiple(
61
- text,
62
- Chunk,
63
- prompt=prompt,
64
- mode="structured", # tool_calls
65
- )
60
+ chunks = await agent.talk.extract_multiple(text, Chunk, prompt=prompt)
66
61
  return Chunks(chunks=chunks)
67
62
 
68
63
  async def split(
@@ -7,8 +7,8 @@ from typing import TYPE_CHECKING, Any, ClassVar, Literal
7
7
  from mkdown import TextChunk
8
8
 
9
9
  from docler.chunkers.base import TextChunker
10
- from docler.configs.chunker_configs import LlamaIndexChunkerConfig
11
10
  from docler.log import get_logger
11
+ from docler_config.chunker_configs import LlamaIndexChunkerConfig
12
12
 
13
13
 
14
14
  if TYPE_CHECKING:
@@ -8,7 +8,7 @@ from mkdown import TextChunk
8
8
 
9
9
  from docler.chunkers.base import TextChunker
10
10
  from docler.chunkers.markdown_chunker.utils import assign_images, split_by_headers
11
- from docler.configs.chunker_configs import MarkdownChunkerConfig
11
+ from docler_config.chunker_configs import MarkdownChunkerConfig
12
12
 
13
13
 
14
14
  if TYPE_CHECKING:
@@ -7,8 +7,8 @@ from typing import TYPE_CHECKING, Any, ClassVar
7
7
  from mkdown import TextChunk
8
8
 
9
9
  from docler.chunkers.base import TextChunker
10
- from docler.configs.chunker_configs import TokenAwareChunkerConfig
11
10
  from docler.log import get_logger
11
+ from docler_config.chunker_configs import TokenAwareChunkerConfig
12
12
 
13
13
 
14
14
  if TYPE_CHECKING:
@@ -1,49 +1,49 @@
1
- """Common types used in the Docler library."""
2
-
3
- from __future__ import annotations
4
-
5
- from typing import Literal
6
-
7
-
8
- VectorId = str | int
9
- SupportedLanguage = Literal["en", "de", "fr", "es", "zh"]
10
- PageRangeString = str
11
-
12
- DEFAULT_CHUNKER_MODEL = "openrouter:openai/o3-mini" # google/gemini-2.0-flash-lite-001
13
- DEFAULT_CONVERTER_MODEL = "google-gla:gemini-2.0-flash"
14
- DEFAULT_ANNOTATOR_MODEL = "google-gla:gemini-2.0-flash"
15
- DEFAULT_IMAGE_ANNOTATOR_MODEL = "google-gla:gemini-2.0-flash"
16
- DEFAULT_PROOF_READER_MODEL = "google-gla:gemini-2.0-flash"
17
-
18
- # Mapping tables for different backends
19
- TESSERACT_CODES: dict[SupportedLanguage, str] = {
20
- "en": "eng",
21
- "de": "deu",
22
- "fr": "fra",
23
- "es": "spa",
24
- "zh": "chi",
25
- }
26
-
27
- MAC_CODES: dict[SupportedLanguage, str] = {
28
- "en": "en-US",
29
- "de": "de-DE",
30
- "fr": "fr-FR",
31
- "es": "es-ES",
32
- "zh": "zh-CN",
33
- }
34
-
35
- RAPID_CODES: dict[SupportedLanguage, str] = {
36
- "en": "english",
37
- "de": "german",
38
- "fr": "french",
39
- "es": "spanish",
40
- "zh": "chinese",
41
- }
42
-
43
- # https://github.com/Goldziher/kreuzberg/blob/main/kreuzberg/_ocr/_paddleocr.py
44
- PADDLE_OCR_CODES: dict[SupportedLanguage, str] = {
45
- "en": "en",
46
- "de": "german",
47
- "fr": "french",
48
- "zh": "ch",
49
- }
1
+ """Common types used in the Docler library."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Literal
6
+
7
+
8
+ VectorId = str | int
9
+ SupportedLanguage = Literal["en", "de", "fr", "es", "zh"]
10
+ PageRangeString = str
11
+
12
+ DEFAULT_CHUNKER_MODEL = "openrouter:openai/o3-mini" # google/gemini-2.0-flash-lite-001
13
+ DEFAULT_CONVERTER_MODEL = "google-gla:gemini-2.0-flash"
14
+ DEFAULT_ANNOTATOR_MODEL = "google-gla:gemini-2.0-flash"
15
+ DEFAULT_IMAGE_ANNOTATOR_MODEL = "google-gla:gemini-2.0-flash"
16
+ DEFAULT_PROOF_READER_MODEL = "google-gla:gemini-2.0-flash"
17
+
18
+ # Mapping tables for different backends
19
+ TESSERACT_CODES: dict[SupportedLanguage, str] = {
20
+ "en": "eng",
21
+ "de": "deu",
22
+ "fr": "fra",
23
+ "es": "spa",
24
+ "zh": "chi",
25
+ }
26
+
27
+ MAC_CODES: dict[SupportedLanguage, str] = {
28
+ "en": "en-US",
29
+ "de": "de-DE",
30
+ "fr": "fr-FR",
31
+ "es": "es-ES",
32
+ "zh": "zh-CN",
33
+ }
34
+
35
+ RAPID_CODES: dict[SupportedLanguage, str] = {
36
+ "en": "english",
37
+ "de": "german",
38
+ "fr": "french",
39
+ "es": "spanish",
40
+ "zh": "chinese",
41
+ }
42
+
43
+ # https://github.com/Goldziher/kreuzberg/blob/main/kreuzberg/_ocr/_paddleocr.py
44
+ PADDLE_OCR_CODES: dict[SupportedLanguage, str] = {
45
+ "en": "en",
46
+ "de": "german",
47
+ "fr": "french",
48
+ "zh": "ch",
49
+ }
@@ -6,9 +6,9 @@ from typing import TYPE_CHECKING
6
6
 
7
7
  import anyenv
8
8
 
9
- from docler.configs.converter_configs import AggregatedConverterConfig
10
9
  from docler.converters.base import DocumentConverter
11
10
  from docler.converters.registry import ConverterRegistry
11
+ from docler_config.converter_configs import AggregatedConverterConfig
12
12
 
13
13
 
14
14
  if TYPE_CHECKING:
@@ -84,11 +84,7 @@ class AggregatedConverter(DocumentConverter[AggregatedConverterConfig]):
84
84
  """
85
85
  self._registry.set_preference(mime_or_extension, converter_name)
86
86
 
87
- async def _convert_async(
88
- self,
89
- data: BytesIO,
90
- mime_type: MimeType,
91
- ) -> ConverterResult:
87
+ async def _convert_async(self, data: BytesIO, mime_type: MimeType) -> ConverterResult:
92
88
  """Delegate conversion to the appropriate converter.
93
89
 
94
90
  Note: We need a filename/extension to select the right converter,