docler 2.0.0__tar.gz → 2.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {docler-2.0.0 → docler-2.1.0}/PKG-INFO +3 -3
- {docler-2.0.0 → docler-2.1.0}/pyproject.toml +4 -3
- {docler-2.0.0 → docler-2.1.0}/src/docler/__main__.py +133 -133
- {docler-2.0.0 → docler-2.1.0}/src/docler/annotators/__init__.py +1 -1
- {docler-2.0.0 → docler-2.1.0}/src/docler/annotators/ai_document_annotator.py +3 -3
- {docler-2.0.0 → docler-2.1.0}/src/docler/annotators/ai_image_annotator.py +11 -5
- {docler-2.0.0 → docler-2.1.0}/src/docler/annotators/base.py +36 -36
- {docler-2.0.0 → docler-2.1.0}/src/docler/chunkers/ai_chunker/chunker.py +4 -9
- {docler-2.0.0 → docler-2.1.0}/src/docler/chunkers/llamaindex_chunker/chunker.py +1 -1
- {docler-2.0.0 → docler-2.1.0}/src/docler/chunkers/markdown_chunker/chunker.py +1 -1
- {docler-2.0.0 → docler-2.1.0}/src/docler/chunkers/token_chunker/chunker.py +1 -1
- {docler-2.0.0 → docler-2.1.0}/src/docler/common_types.py +49 -49
- {docler-2.0.0 → docler-2.1.0}/src/docler/converters/aggregated_converter.py +2 -6
- {docler-2.0.0 → docler-2.1.0}/src/docler/converters/azure_provider/__init__.py +5 -5
- {docler-2.0.0 → docler-2.1.0}/src/docler/converters/azure_provider/provider.py +2 -2
- {docler-2.0.0 → docler-2.1.0}/src/docler/converters/base.py +56 -15
- {docler-2.0.0 → docler-2.1.0}/src/docler/converters/datalab_provider/__init__.py +5 -5
- {docler-2.0.0 → docler-2.1.0}/src/docler/converters/datalab_provider/provider.py +2 -6
- {docler-2.0.0 → docler-2.1.0}/src/docler/converters/dir_converter.py +179 -179
- {docler-2.0.0 → docler-2.1.0}/src/docler/converters/docling_provider/__init__.py +5 -5
- {docler-2.0.0 → docler-2.1.0}/src/docler/converters/docling_provider/provider.py +2 -2
- {docler-2.0.0 → docler-2.1.0}/src/docler/converters/docling_provider/utils.py +82 -82
- {docler-2.0.0 → docler-2.1.0}/src/docler/converters/docling_remote_provider/provider.py +2 -6
- {docler-2.0.0 → docler-2.1.0}/src/docler/converters/exceptions.py +9 -9
- {docler-2.0.0 → docler-2.1.0}/src/docler/converters/llamaparse_provider/__init__.py +5 -5
- {docler-2.0.0 → docler-2.1.0}/src/docler/converters/llamaparse_provider/provider.py +2 -2
- {docler-2.0.0 → docler-2.1.0}/src/docler/converters/llm_provider/__init__.py +5 -5
- {docler-2.0.0 → docler-2.1.0}/src/docler/converters/llm_provider/provider.py +8 -7
- {docler-2.0.0 → docler-2.1.0}/src/docler/converters/marker_provider/__init__.py +5 -5
- {docler-2.0.0 → docler-2.1.0}/src/docler/converters/marker_provider/provider.py +1 -1
- {docler-2.0.0 → docler-2.1.0}/src/docler/converters/markitdown_provider/__init__.py +5 -5
- {docler-2.0.0 → docler-2.1.0}/src/docler/converters/markitdown_provider/provider.py +1 -1
- {docler-2.0.0 → docler-2.1.0}/src/docler/converters/mistral_provider/__init__.py +5 -5
- {docler-2.0.0 → docler-2.1.0}/src/docler/converters/mistral_provider/provider.py +1 -1
- {docler-2.0.0 → docler-2.1.0}/src/docler/converters/upstage_provider/__init__.py +5 -5
- {docler-2.0.0 → docler-2.1.0}/src/docler/converters/upstage_provider/provider.py +2 -2
- {docler-2.0.0 → docler-2.1.0}/src/docler/diffs.py +90 -90
- {docler-2.0.0 → docler-2.1.0}/src/docler/mime_types.py +174 -174
- {docler-2.0.0 → docler-2.1.0}/src/docler/process_runner.py +276 -276
- {docler-2.0.0 → docler-2.1.0}/src/docler/processors/ai_processor.py +8 -10
- {docler-2.0.0 → docler-2.1.0}/src/docler/processors/base.py +35 -35
- {docler-2.0.0 → docler-2.1.0}/src/docler/provider.py +66 -99
- {docler-2.0.0 → docler-2.1.0}/src/docler_api/__init__.py +3 -3
- {docler-2.0.0 → docler-2.1.0}/src/docler_api/main.py +2 -2
- {docler-2.0.0 → docler-2.1.0}/src/docler_api/routes.py +35 -112
- {docler-2.0.0/src/docler/configs → docler-2.1.0/src/docler_config}/__init__.py +91 -91
- {docler-2.0.0/src/docler/configs → docler-2.1.0/src/docler_config}/annotator_configs.py +117 -115
- {docler-2.0.0/src/docler/configs → docler-2.1.0/src/docler_config}/chunker_configs.py +146 -145
- {docler-2.0.0/src/docler/configs → docler-2.1.0/src/docler_config}/converter_configs.py +5 -6
- {docler-2.0.0/src/docler/configs → docler-2.1.0/src/docler_config}/processor_configs.py +79 -79
- docler-2.1.0/src/docler_config/provider.py +44 -0
- {docler-2.0.0 → docler-2.1.0}/src/docler_streamlit/__init__.py +6 -6
- {docler-2.0.0 → docler-2.1.0}/src/docler_streamlit/ocr_app.py +56 -56
- {docler-2.0.0 → docler-2.1.0}/LICENSE +0 -0
- {docler-2.0.0 → docler-2.1.0}/README.md +0 -0
- {docler-2.0.0 → docler-2.1.0}/src/docler/__init__.py +0 -0
- {docler-2.0.0 → docler-2.1.0}/src/docler/chunkers/__init__.py +0 -0
- {docler-2.0.0 → docler-2.1.0}/src/docler/chunkers/ai_chunker/__init__.py +0 -0
- {docler-2.0.0 → docler-2.1.0}/src/docler/chunkers/ai_chunker/models.py +0 -0
- {docler-2.0.0 → docler-2.1.0}/src/docler/chunkers/ai_chunker/utils.py +0 -0
- {docler-2.0.0 → docler-2.1.0}/src/docler/chunkers/base.py +0 -0
- {docler-2.0.0 → docler-2.1.0}/src/docler/chunkers/llamaindex_chunker/__init__.py +0 -0
- {docler-2.0.0 → docler-2.1.0}/src/docler/chunkers/markdown_chunker/__init__.py +0 -0
- {docler-2.0.0 → docler-2.1.0}/src/docler/chunkers/markdown_chunker/utils.py +0 -0
- {docler-2.0.0 → docler-2.1.0}/src/docler/chunkers/token_chunker/__init__.py +0 -0
- {docler-2.0.0 → docler-2.1.0}/src/docler/converters/__init__.py +0 -0
- {docler-2.0.0 → docler-2.1.0}/src/docler/converters/azure_provider/utils.py +0 -0
- {docler-2.0.0 → docler-2.1.0}/src/docler/converters/datalab_provider/utils.py +0 -0
- {docler-2.0.0 → docler-2.1.0}/src/docler/converters/docling_remote_provider/__init__.py +0 -0
- {docler-2.0.0 → docler-2.1.0}/src/docler/converters/docling_remote_provider/utils.py +0 -0
- {docler-2.0.0 → docler-2.1.0}/src/docler/converters/llamaparse_provider/utils.py +0 -0
- {docler-2.0.0 → docler-2.1.0}/src/docler/converters/mistral_provider/utils.py +0 -0
- {docler-2.0.0 → docler-2.1.0}/src/docler/converters/registry.py +0 -0
- {docler-2.0.0 → docler-2.1.0}/src/docler/log.py +0 -0
- {docler-2.0.0 → docler-2.1.0}/src/docler/models.py +0 -0
- {docler-2.0.0 → docler-2.1.0}/src/docler/pdf_utils.py +0 -0
- {docler-2.0.0 → docler-2.1.0}/src/docler/processors/__init__.py +0 -0
- {docler-2.0.0 → docler-2.1.0}/src/docler/py.typed +0 -0
- {docler-2.0.0 → docler-2.1.0}/src/docler/resources/pdf_sample.pdf +0 -0
- {docler-2.0.0 → docler-2.1.0}/src/docler/resources/samplepptx.pptx +0 -0
- {docler-2.0.0 → docler-2.1.0}/src/docler/resources/test.docx +0 -0
- {docler-2.0.0 → docler-2.1.0}/src/docler/resources/test.jpg +0 -0
- {docler-2.0.0 → docler-2.1.0}/src/docler/resources/test.pptx +0 -0
- {docler-2.0.0 → docler-2.1.0}/src/docler/resources/test.xls +0 -0
- {docler-2.0.0 → docler-2.1.0}/src/docler/resources/test.xlsx +0 -0
- {docler-2.0.0 → docler-2.1.0}/src/docler/utils.py +0 -0
- {docler-2.0.0 → docler-2.1.0}/src/docler_api/README.md +0 -0
- {docler-2.0.0 → docler-2.1.0}/src/docler_api/py.typed +0 -0
- {docler-2.0.0 → docler-2.1.0}/src/docler_mcp/__init__.py +0 -0
- {docler-2.0.0 → docler-2.1.0}/src/docler_mcp/py.typed +0 -0
- {docler-2.0.0 → docler-2.1.0}/src/docler_mcp/server.py +0 -0
- {docler-2.0.0 → docler-2.1.0}/src/docler_streamlit/chunk_app.py +0 -0
- {docler-2.0.0 → docler-2.1.0}/src/docler_streamlit/chunkers.py +0 -0
- {docler-2.0.0 → docler-2.1.0}/src/docler_streamlit/converters.py +0 -0
- {docler-2.0.0 → docler-2.1.0}/src/docler_streamlit/py.typed +0 -0
- {docler-2.0.0 → docler-2.1.0}/src/docler_streamlit/utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: docler
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.1.0
|
|
4
4
|
Summary: Abstractions & Tools for OCR / document processing
|
|
5
5
|
Keywords:
|
|
6
6
|
Author: Philipp Temminghoff
|
|
@@ -30,12 +30,12 @@ Requires-Dist: pytest-docker>=3.2.1
|
|
|
30
30
|
Requires-Dist: schemez>=0.0.1
|
|
31
31
|
Requires-Dist: upathtools>=0.4.3
|
|
32
32
|
Requires-Dist: docler[marker,azure,diffs,docling,llama-parse,marker,markitdown,mcp,mistralai,openai,server] ; extra == 'all'
|
|
33
|
-
Requires-Dist:
|
|
33
|
+
Requires-Dist: agentpool[default] ; extra == 'all'
|
|
34
34
|
Requires-Dist: azure-ai-documentintelligence ; extra == 'azure'
|
|
35
35
|
Requires-Dist: diff-match-patch ; extra == 'diffs'
|
|
36
36
|
Requires-Dist: docling[ocrmac,vlm,rapidocr] ; python_full_version < '3.14' and extra == 'docling'
|
|
37
37
|
Requires-Dist: docler[mistralai,diffs,llama-parse,openai,azure] ; extra == 'light'
|
|
38
|
-
Requires-Dist:
|
|
38
|
+
Requires-Dist: agentpool[default] ; extra == 'light'
|
|
39
39
|
Requires-Dist: llama-parse ; python_full_version < '3.14' and extra == 'llama-parse'
|
|
40
40
|
Requires-Dist: marker-pdf ; python_full_version < '3.14' and extra == 'marker'
|
|
41
41
|
Requires-Dist: markitdown[all] ; python_full_version < '3.14' and extra == 'markitdown'
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
[project]
|
|
4
4
|
name = "docler"
|
|
5
|
-
version = "2.
|
|
5
|
+
version = "2.1.0"
|
|
6
6
|
description = "Abstractions & Tools for OCR / document processing"
|
|
7
7
|
readme = "README.md"
|
|
8
8
|
requires-python = ">=3.13"
|
|
@@ -54,7 +54,7 @@ docler = "docler.__main__:cli"
|
|
|
54
54
|
[project.optional-dependencies]
|
|
55
55
|
all = [
|
|
56
56
|
"docler[marker,azure,diffs,docling,llama-parse,marker,markitdown,mcp,mistralai,openai,server]",
|
|
57
|
-
"
|
|
57
|
+
"agentpool[default]"
|
|
58
58
|
]
|
|
59
59
|
azure = ["azure-ai-documentintelligence"]
|
|
60
60
|
diffs = ["diff-match-patch"]
|
|
@@ -63,7 +63,7 @@ docling = [
|
|
|
63
63
|
] # tesserocr
|
|
64
64
|
light = [
|
|
65
65
|
"docler[mistralai,diffs,llama-parse,openai,azure]",
|
|
66
|
-
"
|
|
66
|
+
"agentpool[default]"
|
|
67
67
|
]
|
|
68
68
|
llama-parse = ["llama-parse; python_version < '3.14'"]
|
|
69
69
|
marker = ["marker-pdf; python_version < '3.14'"]
|
|
@@ -316,6 +316,7 @@ default-groups = ["dev", "lint", "docs"]
|
|
|
316
316
|
[tool.uv.build-backend]
|
|
317
317
|
module-name = [
|
|
318
318
|
"docler",
|
|
319
|
+
"docler_config",
|
|
319
320
|
"docler_streamlit",
|
|
320
321
|
"docler_api",
|
|
321
322
|
"docler_mcp",
|
|
@@ -1,133 +1,133 @@
|
|
|
1
|
-
"""Command line interface for Docler document converter."""
|
|
2
|
-
|
|
3
|
-
from __future__ import annotations
|
|
4
|
-
|
|
5
|
-
from enum import Enum
|
|
6
|
-
import os
|
|
7
|
-
import pathlib
|
|
8
|
-
import subprocess
|
|
9
|
-
import sys
|
|
10
|
-
|
|
11
|
-
import typer
|
|
12
|
-
|
|
13
|
-
from docler.log import get_logger
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
cli = typer.Typer(help="Docler document converter CLI", no_args_is_help=True)
|
|
17
|
-
|
|
18
|
-
logger = get_logger(__name__)
|
|
19
|
-
|
|
20
|
-
PACKAGE_DIR = pathlib.Path(__file__).parent.parent / "docler_streamlit"
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
class TransportType(str, Enum):
|
|
24
|
-
"""Transport protocol types for MCP server."""
|
|
25
|
-
|
|
26
|
-
STDIO = "stdio"
|
|
27
|
-
SSE = "sse"
|
|
28
|
-
STREAMABLE_HTTP = "streamable-http"
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
@cli.command()
|
|
32
|
-
def flow() -> None:
|
|
33
|
-
"""Start the Streamlit web interface."""
|
|
34
|
-
app_path = str(PACKAGE_DIR / "app.py")
|
|
35
|
-
try:
|
|
36
|
-
cmd = [sys.executable, "-m", "streamlit", "run", app_path]
|
|
37
|
-
subprocess.run(cmd, env=os.environ.copy(), check=True)
|
|
38
|
-
except subprocess.CalledProcessError as e:
|
|
39
|
-
# msg = f"Failed to start Streamlit: {e}"
|
|
40
|
-
raise typer.Exit(1) from e
|
|
41
|
-
except KeyboardInterrupt:
|
|
42
|
-
logger.info("Shutting down...")
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
@cli.command("chunk_ui")
|
|
46
|
-
def chunk_ui() -> None:
|
|
47
|
-
"""Start the Streamlit web interface."""
|
|
48
|
-
app_path = str(PACKAGE_DIR / "chunk_app.py")
|
|
49
|
-
try:
|
|
50
|
-
cmd = [sys.executable, "-m", "streamlit", "run", app_path]
|
|
51
|
-
subprocess.run(cmd, env=os.environ.copy(), check=True)
|
|
52
|
-
except subprocess.CalledProcessError as e:
|
|
53
|
-
# msg = f"Failed to start Streamlit: {e}"
|
|
54
|
-
raise typer.Exit(1) from e
|
|
55
|
-
except KeyboardInterrupt:
|
|
56
|
-
logger.info("Shutting down...")
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
@cli.command("ocr_ui")
|
|
60
|
-
def ocr_ui() -> None:
|
|
61
|
-
"""Start the Streamlit web interface."""
|
|
62
|
-
app_path = str(PACKAGE_DIR / "ocr_app.py")
|
|
63
|
-
try:
|
|
64
|
-
cmd = [sys.executable, "-m", "streamlit", "run", app_path]
|
|
65
|
-
subprocess.run(cmd, env=os.environ.copy(), check=True)
|
|
66
|
-
except subprocess.CalledProcessError as e:
|
|
67
|
-
# msg = f"Failed to start Streamlit: {e}"
|
|
68
|
-
raise typer.Exit(1) from e
|
|
69
|
-
except KeyboardInterrupt:
|
|
70
|
-
logger.info("Shutting down...")
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
@cli.command()
|
|
74
|
-
def api(
|
|
75
|
-
host: str = typer.Option("0.0.0.0", help="Host to bind to"),
|
|
76
|
-
port: int = typer.Option(8000, help="Port to bind to"),
|
|
77
|
-
log_level: str = typer.Option("info", help="Log level"),
|
|
78
|
-
reload: bool = typer.Option(False, help="Enable auto-reload on file changes"),
|
|
79
|
-
) -> None:
|
|
80
|
-
"""Start the Docler API server."""
|
|
81
|
-
try:
|
|
82
|
-
# Lazy import to avoid unnecessary dependencies if not using the API
|
|
83
|
-
import uvicorn
|
|
84
|
-
|
|
85
|
-
from docler_api.main import app
|
|
86
|
-
|
|
87
|
-
uvicorn.run(
|
|
88
|
-
app,
|
|
89
|
-
host=host,
|
|
90
|
-
port=port,
|
|
91
|
-
log_level=log_level,
|
|
92
|
-
reload=reload,
|
|
93
|
-
)
|
|
94
|
-
except ImportError as e:
|
|
95
|
-
msg = f"Failed to import API components: {e}. Is the 'server' extra installed?"
|
|
96
|
-
logger.exception(msg)
|
|
97
|
-
raise typer.Exit(1) from e
|
|
98
|
-
except KeyboardInterrupt:
|
|
99
|
-
logger.info("Shutting down...")
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
@cli.command()
|
|
103
|
-
def mcp(
|
|
104
|
-
transport: TransportType = typer.Option( # noqa: B008
|
|
105
|
-
TransportType.STDIO, help="Transport protocol to use"
|
|
106
|
-
),
|
|
107
|
-
host: str = typer.Option("0.0.0.0", help="Host to bind the server to"),
|
|
108
|
-
port: int = typer.Option(8000, help="Port to bind the server to"),
|
|
109
|
-
) -> None:
|
|
110
|
-
"""Start the MCP server."""
|
|
111
|
-
try:
|
|
112
|
-
# Lazy import to avoid unnecessary dependencies if not using MCP
|
|
113
|
-
from docler_mcp.server import mcp as mcp_server
|
|
114
|
-
|
|
115
|
-
if transport == TransportType.STDIO:
|
|
116
|
-
mcp_server.run(transport=transport.value)
|
|
117
|
-
else:
|
|
118
|
-
mcp_server.run(transport=transport.value, host=host, port=port)
|
|
119
|
-
except ImportError as e:
|
|
120
|
-
msg = f"Failed to import MCP components: {e}. Is the 'mcp' extra installed?"
|
|
121
|
-
logger.exception(msg)
|
|
122
|
-
raise typer.Exit(1) from e
|
|
123
|
-
except KeyboardInterrupt:
|
|
124
|
-
logger.info("Shutting down...")
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
def main() -> None:
|
|
128
|
-
"""Main entry point."""
|
|
129
|
-
cli()
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
if __name__ == "__main__":
|
|
133
|
-
main()
|
|
1
|
+
"""Command line interface for Docler document converter."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from enum import Enum
|
|
6
|
+
import os
|
|
7
|
+
import pathlib
|
|
8
|
+
import subprocess
|
|
9
|
+
import sys
|
|
10
|
+
|
|
11
|
+
import typer
|
|
12
|
+
|
|
13
|
+
from docler.log import get_logger
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
cli = typer.Typer(help="Docler document converter CLI", no_args_is_help=True)
|
|
17
|
+
|
|
18
|
+
logger = get_logger(__name__)
|
|
19
|
+
|
|
20
|
+
PACKAGE_DIR = pathlib.Path(__file__).parent.parent / "docler_streamlit"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class TransportType(str, Enum):
|
|
24
|
+
"""Transport protocol types for MCP server."""
|
|
25
|
+
|
|
26
|
+
STDIO = "stdio"
|
|
27
|
+
SSE = "sse"
|
|
28
|
+
STREAMABLE_HTTP = "streamable-http"
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@cli.command()
|
|
32
|
+
def flow() -> None:
|
|
33
|
+
"""Start the Streamlit web interface."""
|
|
34
|
+
app_path = str(PACKAGE_DIR / "app.py")
|
|
35
|
+
try:
|
|
36
|
+
cmd = [sys.executable, "-m", "streamlit", "run", app_path]
|
|
37
|
+
subprocess.run(cmd, env=os.environ.copy(), check=True)
|
|
38
|
+
except subprocess.CalledProcessError as e:
|
|
39
|
+
# msg = f"Failed to start Streamlit: {e}"
|
|
40
|
+
raise typer.Exit(1) from e
|
|
41
|
+
except KeyboardInterrupt:
|
|
42
|
+
logger.info("Shutting down...")
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@cli.command("chunk_ui")
|
|
46
|
+
def chunk_ui() -> None:
|
|
47
|
+
"""Start the Streamlit web interface."""
|
|
48
|
+
app_path = str(PACKAGE_DIR / "chunk_app.py")
|
|
49
|
+
try:
|
|
50
|
+
cmd = [sys.executable, "-m", "streamlit", "run", app_path]
|
|
51
|
+
subprocess.run(cmd, env=os.environ.copy(), check=True)
|
|
52
|
+
except subprocess.CalledProcessError as e:
|
|
53
|
+
# msg = f"Failed to start Streamlit: {e}"
|
|
54
|
+
raise typer.Exit(1) from e
|
|
55
|
+
except KeyboardInterrupt:
|
|
56
|
+
logger.info("Shutting down...")
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@cli.command("ocr_ui")
|
|
60
|
+
def ocr_ui() -> None:
|
|
61
|
+
"""Start the Streamlit web interface."""
|
|
62
|
+
app_path = str(PACKAGE_DIR / "ocr_app.py")
|
|
63
|
+
try:
|
|
64
|
+
cmd = [sys.executable, "-m", "streamlit", "run", app_path]
|
|
65
|
+
subprocess.run(cmd, env=os.environ.copy(), check=True)
|
|
66
|
+
except subprocess.CalledProcessError as e:
|
|
67
|
+
# msg = f"Failed to start Streamlit: {e}"
|
|
68
|
+
raise typer.Exit(1) from e
|
|
69
|
+
except KeyboardInterrupt:
|
|
70
|
+
logger.info("Shutting down...")
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
@cli.command()
|
|
74
|
+
def api(
|
|
75
|
+
host: str = typer.Option("0.0.0.0", help="Host to bind to"),
|
|
76
|
+
port: int = typer.Option(8000, help="Port to bind to"),
|
|
77
|
+
log_level: str = typer.Option("info", help="Log level"),
|
|
78
|
+
reload: bool = typer.Option(False, help="Enable auto-reload on file changes"),
|
|
79
|
+
) -> None:
|
|
80
|
+
"""Start the Docler API server."""
|
|
81
|
+
try:
|
|
82
|
+
# Lazy import to avoid unnecessary dependencies if not using the API
|
|
83
|
+
import uvicorn
|
|
84
|
+
|
|
85
|
+
from docler_api.main import app
|
|
86
|
+
|
|
87
|
+
uvicorn.run(
|
|
88
|
+
app,
|
|
89
|
+
host=host,
|
|
90
|
+
port=port,
|
|
91
|
+
log_level=log_level,
|
|
92
|
+
reload=reload,
|
|
93
|
+
)
|
|
94
|
+
except ImportError as e:
|
|
95
|
+
msg = f"Failed to import API components: {e}. Is the 'server' extra installed?"
|
|
96
|
+
logger.exception(msg)
|
|
97
|
+
raise typer.Exit(1) from e
|
|
98
|
+
except KeyboardInterrupt:
|
|
99
|
+
logger.info("Shutting down...")
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
@cli.command()
|
|
103
|
+
def mcp(
|
|
104
|
+
transport: TransportType = typer.Option( # noqa: B008
|
|
105
|
+
TransportType.STDIO, help="Transport protocol to use"
|
|
106
|
+
),
|
|
107
|
+
host: str = typer.Option("0.0.0.0", help="Host to bind the server to"),
|
|
108
|
+
port: int = typer.Option(8000, help="Port to bind the server to"),
|
|
109
|
+
) -> None:
|
|
110
|
+
"""Start the MCP server."""
|
|
111
|
+
try:
|
|
112
|
+
# Lazy import to avoid unnecessary dependencies if not using MCP
|
|
113
|
+
from docler_mcp.server import mcp as mcp_server
|
|
114
|
+
|
|
115
|
+
if transport == TransportType.STDIO:
|
|
116
|
+
mcp_server.run(transport=transport.value)
|
|
117
|
+
else:
|
|
118
|
+
mcp_server.run(transport=transport.value, host=host, port=port)
|
|
119
|
+
except ImportError as e:
|
|
120
|
+
msg = f"Failed to import MCP components: {e}. Is the 'mcp' extra installed?"
|
|
121
|
+
logger.exception(msg)
|
|
122
|
+
raise typer.Exit(1) from e
|
|
123
|
+
except KeyboardInterrupt:
|
|
124
|
+
logger.info("Shutting down...")
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def main() -> None:
|
|
128
|
+
"""Main entry point."""
|
|
129
|
+
cli()
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
if __name__ == "__main__":
|
|
133
|
+
main()
|
|
@@ -1 +1 @@
|
|
|
1
|
-
"""Annotators for Documents."""
|
|
1
|
+
"""Annotators for Documents."""
|
|
@@ -10,7 +10,7 @@ from schemez import Schema
|
|
|
10
10
|
|
|
11
11
|
from docler.annotators.base import Annotator
|
|
12
12
|
from docler.common_types import DEFAULT_ANNOTATOR_MODEL
|
|
13
|
-
from
|
|
13
|
+
from docler_config.annotator_configs import (
|
|
14
14
|
DEFAULT_DOC_PROMPT_TEMPLATE,
|
|
15
15
|
DEFAULT_DOC_SYSTEM_PROMPT,
|
|
16
16
|
AIDocumentAnnotatorConfig,
|
|
@@ -46,7 +46,7 @@ class AIDocumentAnnotator[TMetadata: Schema = DefaultMetadata](
|
|
|
46
46
|
"""
|
|
47
47
|
|
|
48
48
|
Config = AIDocumentAnnotatorConfig
|
|
49
|
-
REQUIRED_PACKAGES: ClassVar = {"
|
|
49
|
+
REQUIRED_PACKAGES: ClassVar = {"agentpool"}
|
|
50
50
|
|
|
51
51
|
def __init__(
|
|
52
52
|
self,
|
|
@@ -84,7 +84,7 @@ class AIDocumentAnnotator[TMetadata: Schema = DefaultMetadata](
|
|
|
84
84
|
Returns:
|
|
85
85
|
Document with enhanced metadata
|
|
86
86
|
"""
|
|
87
|
-
from
|
|
87
|
+
from agentpool import Agent
|
|
88
88
|
|
|
89
89
|
agent = Agent(
|
|
90
90
|
model=self.model,
|
|
@@ -10,7 +10,7 @@ from schemez import Schema
|
|
|
10
10
|
|
|
11
11
|
from docler.annotators.base import Annotator
|
|
12
12
|
from docler.common_types import DEFAULT_IMAGE_ANNOTATOR_MODEL
|
|
13
|
-
from
|
|
13
|
+
from docler_config.annotator_configs import (
|
|
14
14
|
DEFAULT_IMAGE_PROMPT_TEMPLATE,
|
|
15
15
|
DEFAULT_IMAGE_SYSTEM_PROMPT,
|
|
16
16
|
AIImageAnnotatorConfig,
|
|
@@ -53,7 +53,7 @@ class AIImageAnnotator[TMetadata: Schema = DefaultImageMetadata](Annotator[AIIma
|
|
|
53
53
|
|
|
54
54
|
Config = AIImageAnnotatorConfig
|
|
55
55
|
|
|
56
|
-
REQUIRED_PACKAGES: ClassVar = {"
|
|
56
|
+
REQUIRED_PACKAGES: ClassVar = {"agentpool"}
|
|
57
57
|
|
|
58
58
|
def __init__(
|
|
59
59
|
self,
|
|
@@ -88,13 +88,19 @@ class AIImageAnnotator[TMetadata: Schema = DefaultImageMetadata](Annotator[AIIma
|
|
|
88
88
|
Returns:
|
|
89
89
|
Image with added description and metadata
|
|
90
90
|
"""
|
|
91
|
-
|
|
91
|
+
import base64
|
|
92
|
+
|
|
93
|
+
from agentpool import Agent
|
|
94
|
+
from pydantic_ai import BinaryImage
|
|
92
95
|
|
|
93
96
|
if image.description and image.metadata:
|
|
94
97
|
return image
|
|
95
98
|
|
|
96
|
-
|
|
97
|
-
|
|
99
|
+
if isinstance(image.content, bytes):
|
|
100
|
+
data = image.content
|
|
101
|
+
else:
|
|
102
|
+
data = base64.b64decode(image.content)
|
|
103
|
+
img_content = BinaryImage(data=data, media_type=image.mime_type)
|
|
98
104
|
agent = Agent(
|
|
99
105
|
model=self.model,
|
|
100
106
|
system_prompt=self.system_prompt,
|
|
@@ -1,36 +1,36 @@
|
|
|
1
|
-
"""Base classes for text chunking implementations."""
|
|
2
|
-
|
|
3
|
-
from __future__ import annotations
|
|
4
|
-
|
|
5
|
-
from abc import ABC, abstractmethod
|
|
6
|
-
from typing import TYPE_CHECKING, ClassVar
|
|
7
|
-
|
|
8
|
-
from pydantic import BaseModel
|
|
9
|
-
|
|
10
|
-
from docler.provider import BaseProvider
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
if TYPE_CHECKING:
|
|
14
|
-
from docler.
|
|
15
|
-
from
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
class Annotator[TConfig: BaseModel](ABC, BaseProvider[TConfig]):
|
|
19
|
-
"""Base class for chunk annotation processors."""
|
|
20
|
-
|
|
21
|
-
Config: ClassVar[type[BaseAnnotatorConfig]]
|
|
22
|
-
|
|
23
|
-
@abstractmethod
|
|
24
|
-
async def annotate(
|
|
25
|
-
self,
|
|
26
|
-
chunked_doc: ChunkedDocument,
|
|
27
|
-
) -> ChunkedDocument:
|
|
28
|
-
"""Annotate a chunked document with additional metadata.
|
|
29
|
-
|
|
30
|
-
Args:
|
|
31
|
-
chunked_doc: Chunked document containing the original content and chunks
|
|
32
|
-
|
|
33
|
-
Returns:
|
|
34
|
-
Chunked document with annotated chunks
|
|
35
|
-
"""
|
|
36
|
-
raise NotImplementedError
|
|
1
|
+
"""Base classes for text chunking implementations."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from abc import ABC, abstractmethod
|
|
6
|
+
from typing import TYPE_CHECKING, ClassVar
|
|
7
|
+
|
|
8
|
+
from pydantic import BaseModel
|
|
9
|
+
|
|
10
|
+
from docler.provider import BaseProvider
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
if TYPE_CHECKING:
|
|
14
|
+
from docler.models import ChunkedDocument
|
|
15
|
+
from docler_config.annotator_configs import BaseAnnotatorConfig
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class Annotator[TConfig: BaseModel](ABC, BaseProvider[TConfig]):
|
|
19
|
+
"""Base class for chunk annotation processors."""
|
|
20
|
+
|
|
21
|
+
Config: ClassVar[type[BaseAnnotatorConfig]]
|
|
22
|
+
|
|
23
|
+
@abstractmethod
|
|
24
|
+
async def annotate(
|
|
25
|
+
self,
|
|
26
|
+
chunked_doc: ChunkedDocument,
|
|
27
|
+
) -> ChunkedDocument:
|
|
28
|
+
"""Annotate a chunked document with additional metadata.
|
|
29
|
+
|
|
30
|
+
Args:
|
|
31
|
+
chunked_doc: Chunked document containing the original content and chunks
|
|
32
|
+
|
|
33
|
+
Returns:
|
|
34
|
+
Chunked document with annotated chunks
|
|
35
|
+
"""
|
|
36
|
+
raise NotImplementedError
|
|
@@ -8,7 +8,7 @@ from docler.chunkers.ai_chunker.models import Chunk, Chunks
|
|
|
8
8
|
from docler.chunkers.ai_chunker.utils import add_line_numbers, create_text_chunk
|
|
9
9
|
from docler.chunkers.base import TextChunker
|
|
10
10
|
from docler.common_types import DEFAULT_CHUNKER_MODEL
|
|
11
|
-
from
|
|
11
|
+
from docler_config.chunker_configs import (
|
|
12
12
|
DEFAULT_CHUNKER_SYSTEM_PROMPT,
|
|
13
13
|
DEFAULT_CHUNKER_USER_TEMPLATE,
|
|
14
14
|
AiChunkerConfig,
|
|
@@ -23,7 +23,7 @@ class AIChunker(TextChunker[AiChunkerConfig]):
|
|
|
23
23
|
"""LLM-based document chunker."""
|
|
24
24
|
|
|
25
25
|
NAME = "ai"
|
|
26
|
-
REQUIRED_PACKAGES: ClassVar = {"
|
|
26
|
+
REQUIRED_PACKAGES: ClassVar = {"agentpool"}
|
|
27
27
|
Config = AiChunkerConfig
|
|
28
28
|
|
|
29
29
|
def __init__(
|
|
@@ -46,7 +46,7 @@ class AIChunker(TextChunker[AiChunkerConfig]):
|
|
|
46
46
|
|
|
47
47
|
async def _get_chunks(self, text: str) -> Chunks:
|
|
48
48
|
"""Get chunk definitions from LLM."""
|
|
49
|
-
from
|
|
49
|
+
from agentpool import Agent
|
|
50
50
|
|
|
51
51
|
numbered_text = add_line_numbers(text)
|
|
52
52
|
# agent: llmling_agent.StructuredAgent[None, Chunks] = llmling_agent.Agent[None](
|
|
@@ -57,12 +57,7 @@ class AIChunker(TextChunker[AiChunkerConfig]):
|
|
|
57
57
|
# response = await agent.run(prompt)
|
|
58
58
|
agent: Agent[None] = Agent[None](model=self.model, system_prompt=self.system_prompt)
|
|
59
59
|
prompt = self.user_prompt.format(numbered_text=numbered_text)
|
|
60
|
-
chunks = await agent.talk.extract_multiple(
|
|
61
|
-
text,
|
|
62
|
-
Chunk,
|
|
63
|
-
prompt=prompt,
|
|
64
|
-
mode="structured", # tool_calls
|
|
65
|
-
)
|
|
60
|
+
chunks = await agent.talk.extract_multiple(text, Chunk, prompt=prompt)
|
|
66
61
|
return Chunks(chunks=chunks)
|
|
67
62
|
|
|
68
63
|
async def split(
|
|
@@ -7,8 +7,8 @@ from typing import TYPE_CHECKING, Any, ClassVar, Literal
|
|
|
7
7
|
from mkdown import TextChunk
|
|
8
8
|
|
|
9
9
|
from docler.chunkers.base import TextChunker
|
|
10
|
-
from docler.configs.chunker_configs import LlamaIndexChunkerConfig
|
|
11
10
|
from docler.log import get_logger
|
|
11
|
+
from docler_config.chunker_configs import LlamaIndexChunkerConfig
|
|
12
12
|
|
|
13
13
|
|
|
14
14
|
if TYPE_CHECKING:
|
|
@@ -8,7 +8,7 @@ from mkdown import TextChunk
|
|
|
8
8
|
|
|
9
9
|
from docler.chunkers.base import TextChunker
|
|
10
10
|
from docler.chunkers.markdown_chunker.utils import assign_images, split_by_headers
|
|
11
|
-
from
|
|
11
|
+
from docler_config.chunker_configs import MarkdownChunkerConfig
|
|
12
12
|
|
|
13
13
|
|
|
14
14
|
if TYPE_CHECKING:
|
|
@@ -7,8 +7,8 @@ from typing import TYPE_CHECKING, Any, ClassVar
|
|
|
7
7
|
from mkdown import TextChunk
|
|
8
8
|
|
|
9
9
|
from docler.chunkers.base import TextChunker
|
|
10
|
-
from docler.configs.chunker_configs import TokenAwareChunkerConfig
|
|
11
10
|
from docler.log import get_logger
|
|
11
|
+
from docler_config.chunker_configs import TokenAwareChunkerConfig
|
|
12
12
|
|
|
13
13
|
|
|
14
14
|
if TYPE_CHECKING:
|
|
@@ -1,49 +1,49 @@
|
|
|
1
|
-
"""Common types used in the Docler library."""
|
|
2
|
-
|
|
3
|
-
from __future__ import annotations
|
|
4
|
-
|
|
5
|
-
from typing import Literal
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
VectorId = str | int
|
|
9
|
-
SupportedLanguage = Literal["en", "de", "fr", "es", "zh"]
|
|
10
|
-
PageRangeString = str
|
|
11
|
-
|
|
12
|
-
DEFAULT_CHUNKER_MODEL = "openrouter:openai/o3-mini" # google/gemini-2.0-flash-lite-001
|
|
13
|
-
DEFAULT_CONVERTER_MODEL = "google-gla:gemini-2.0-flash"
|
|
14
|
-
DEFAULT_ANNOTATOR_MODEL = "google-gla:gemini-2.0-flash"
|
|
15
|
-
DEFAULT_IMAGE_ANNOTATOR_MODEL = "google-gla:gemini-2.0-flash"
|
|
16
|
-
DEFAULT_PROOF_READER_MODEL = "google-gla:gemini-2.0-flash"
|
|
17
|
-
|
|
18
|
-
# Mapping tables for different backends
|
|
19
|
-
TESSERACT_CODES: dict[SupportedLanguage, str] = {
|
|
20
|
-
"en": "eng",
|
|
21
|
-
"de": "deu",
|
|
22
|
-
"fr": "fra",
|
|
23
|
-
"es": "spa",
|
|
24
|
-
"zh": "chi",
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
MAC_CODES: dict[SupportedLanguage, str] = {
|
|
28
|
-
"en": "en-US",
|
|
29
|
-
"de": "de-DE",
|
|
30
|
-
"fr": "fr-FR",
|
|
31
|
-
"es": "es-ES",
|
|
32
|
-
"zh": "zh-CN",
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
RAPID_CODES: dict[SupportedLanguage, str] = {
|
|
36
|
-
"en": "english",
|
|
37
|
-
"de": "german",
|
|
38
|
-
"fr": "french",
|
|
39
|
-
"es": "spanish",
|
|
40
|
-
"zh": "chinese",
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
# https://github.com/Goldziher/kreuzberg/blob/main/kreuzberg/_ocr/_paddleocr.py
|
|
44
|
-
PADDLE_OCR_CODES: dict[SupportedLanguage, str] = {
|
|
45
|
-
"en": "en",
|
|
46
|
-
"de": "german",
|
|
47
|
-
"fr": "french",
|
|
48
|
-
"zh": "ch",
|
|
49
|
-
}
|
|
1
|
+
"""Common types used in the Docler library."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Literal
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
VectorId = str | int
|
|
9
|
+
SupportedLanguage = Literal["en", "de", "fr", "es", "zh"]
|
|
10
|
+
PageRangeString = str
|
|
11
|
+
|
|
12
|
+
DEFAULT_CHUNKER_MODEL = "openrouter:openai/o3-mini" # google/gemini-2.0-flash-lite-001
|
|
13
|
+
DEFAULT_CONVERTER_MODEL = "google-gla:gemini-2.0-flash"
|
|
14
|
+
DEFAULT_ANNOTATOR_MODEL = "google-gla:gemini-2.0-flash"
|
|
15
|
+
DEFAULT_IMAGE_ANNOTATOR_MODEL = "google-gla:gemini-2.0-flash"
|
|
16
|
+
DEFAULT_PROOF_READER_MODEL = "google-gla:gemini-2.0-flash"
|
|
17
|
+
|
|
18
|
+
# Mapping tables for different backends
|
|
19
|
+
TESSERACT_CODES: dict[SupportedLanguage, str] = {
|
|
20
|
+
"en": "eng",
|
|
21
|
+
"de": "deu",
|
|
22
|
+
"fr": "fra",
|
|
23
|
+
"es": "spa",
|
|
24
|
+
"zh": "chi",
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
MAC_CODES: dict[SupportedLanguage, str] = {
|
|
28
|
+
"en": "en-US",
|
|
29
|
+
"de": "de-DE",
|
|
30
|
+
"fr": "fr-FR",
|
|
31
|
+
"es": "es-ES",
|
|
32
|
+
"zh": "zh-CN",
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
RAPID_CODES: dict[SupportedLanguage, str] = {
|
|
36
|
+
"en": "english",
|
|
37
|
+
"de": "german",
|
|
38
|
+
"fr": "french",
|
|
39
|
+
"es": "spanish",
|
|
40
|
+
"zh": "chinese",
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
# https://github.com/Goldziher/kreuzberg/blob/main/kreuzberg/_ocr/_paddleocr.py
|
|
44
|
+
PADDLE_OCR_CODES: dict[SupportedLanguage, str] = {
|
|
45
|
+
"en": "en",
|
|
46
|
+
"de": "german",
|
|
47
|
+
"fr": "french",
|
|
48
|
+
"zh": "ch",
|
|
49
|
+
}
|
|
@@ -6,9 +6,9 @@ from typing import TYPE_CHECKING
|
|
|
6
6
|
|
|
7
7
|
import anyenv
|
|
8
8
|
|
|
9
|
-
from docler.configs.converter_configs import AggregatedConverterConfig
|
|
10
9
|
from docler.converters.base import DocumentConverter
|
|
11
10
|
from docler.converters.registry import ConverterRegistry
|
|
11
|
+
from docler_config.converter_configs import AggregatedConverterConfig
|
|
12
12
|
|
|
13
13
|
|
|
14
14
|
if TYPE_CHECKING:
|
|
@@ -84,11 +84,7 @@ class AggregatedConverter(DocumentConverter[AggregatedConverterConfig]):
|
|
|
84
84
|
"""
|
|
85
85
|
self._registry.set_preference(mime_or_extension, converter_name)
|
|
86
86
|
|
|
87
|
-
async def _convert_async(
|
|
88
|
-
self,
|
|
89
|
-
data: BytesIO,
|
|
90
|
-
mime_type: MimeType,
|
|
91
|
-
) -> ConverterResult:
|
|
87
|
+
async def _convert_async(self, data: BytesIO, mime_type: MimeType) -> ConverterResult:
|
|
92
88
|
"""Delegate conversion to the appropriate converter.
|
|
93
89
|
|
|
94
90
|
Note: We need a filename/extension to select the right converter,
|