deepdoc-lib 0.2.2__tar.gz → 0.2.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/.env.example +1 -1
- deepdoc_lib-0.2.5/.github/workflows/ci.yml +105 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/PKG-INFO +12 -3
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/README.md +8 -0
- deepdoc_lib-0.2.5/UPSTREAM_SYNC.md +86 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/__init__.py +1 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/_version.py +3 -3
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/depend/simple_cv_model.py +30 -10
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/__init__.py +2 -0
- deepdoc_lib-0.2.5/deepdoc/parser/epub_parser.py +139 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/excel_parser.py +78 -25
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/html_parser.py +77 -14
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/markdown_parser.py +240 -34
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/mineru_parser.py +42 -22
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/pdf_parser.py +388 -41
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/ppt_parser.py +9 -3
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/tcadp_parser.py +43 -23
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/utils.py +1 -1
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/vision/layout_recognizer.py +35 -24
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/vision/ocr.py +8 -2
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/vision/operators.py +1 -1
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/vision/table_structure_recognizer.py +6 -1
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc_lib.egg-info/PKG-INFO +12 -3
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc_lib.egg-info/SOURCES.txt +2 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc_lib.egg-info/requires.txt +4 -2
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc_lib.egg-info/scm_file_list.json +72 -67
- deepdoc_lib-0.2.5/deepdoc_lib.egg-info/scm_version.json +8 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/pyproject.toml +2 -2
- deepdoc_lib-0.2.2/.github/workflows/ci.yml +0 -31
- deepdoc_lib-0.2.2/deepdoc_lib.egg-info/scm_version.json +0 -8
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/.github/workflows/publish.yml +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/LICENSE +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/MANIFEST.in +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/README.md +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/README_zh.md +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/common/__init__.py +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/common/config_utils.py +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/common/connection_utils.py +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/common/file_utils.py +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/common/misc_utils.py +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/common/model_store.py +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/common/settings.py +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/common/tiktoken_cache.py +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/common/token_utils.py +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/config.py +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/depend/find_codec.py +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/depend/nltk_manager.py +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/depend/prompts/vision_llm_describe_prompt.md +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/depend/prompts/vision_llm_figure_describe_prompt.md +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/depend/prompts.py +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/depend/rag_tokenizer.py +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/depend/surname.py +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/depend/timeout.py +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/depend/vision_llm_chunk.py +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/dict/README.md +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/dict/huqie.txt +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/download_models.py +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/llm_adapter/__init__.py +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/llm_adapter/adapter.py +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/llm_adapter/utils.py +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/llm_adapter/vision.py +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/docling_parser.py +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/docx_parser.py +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/figure_parser.py +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/json_parser.py +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/resume/__init__.py +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/resume/entities/__init__.py +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/resume/entities/corporations.py +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/resume/entities/degrees.py +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/resume/entities/industries.py +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/resume/entities/regions.py +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/resume/entities/res/corp.tks.freq.json +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/resume/entities/res/corp_baike_len.csv +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/resume/entities/res/corp_tag.json +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/resume/entities/res/good_corp.json +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/resume/entities/res/good_sch.json +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/resume/entities/res/school.rank.csv +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/resume/entities/res/schools.csv +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/resume/entities/schools.py +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/resume/step_one.py +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/resume/step_two.py +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/txt_parser.py +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/vision/__init__.py +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/vision/postprocess.py +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/vision/recognizer.py +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/vision/seeit.py +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/vision/t_ocr.py +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/vision/t_recognizer.py +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc_config.example.yaml +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc_lib.egg-info/dependency_links.txt +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc_lib.egg-info/entry_points.txt +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc_lib.egg-info/top_level.txt +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/scripts/download_models.py +0 -0
- {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/setup.cfg +0 -0
|
@@ -29,7 +29,7 @@ DEEPDOC_VISION_BASE_URL=
|
|
|
29
29
|
# openai - OpenAI GPT-4 Vision
|
|
30
30
|
# qwen - 阿里云通义千问
|
|
31
31
|
# zhipu - 智谱AI
|
|
32
|
-
# gemini - Google Gemini
|
|
32
|
+
# gemini - Google Gemini (需安装: pip install deepdoc-lib[gemini])
|
|
33
33
|
# ollama - 本地 Ollama 服务
|
|
34
34
|
# anthropic - Anthropic Claude (需额外配置)
|
|
35
35
|
DEEPDOC_VISION_PROVIDER=qwen
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
name: Tests
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
pull_request:
|
|
6
|
+
workflow_dispatch:
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
pytest:
|
|
10
|
+
name: Pytest (${{ matrix.python-version }})
|
|
11
|
+
runs-on: ubuntu-latest
|
|
12
|
+
strategy:
|
|
13
|
+
matrix:
|
|
14
|
+
python-version: ["3.10", "3.11", "3.12", "3.13"]
|
|
15
|
+
|
|
16
|
+
steps:
|
|
17
|
+
- uses: actions/checkout@v4
|
|
18
|
+
|
|
19
|
+
- name: Set up Python
|
|
20
|
+
uses: actions/setup-python@v5
|
|
21
|
+
with:
|
|
22
|
+
python-version: ${{ matrix.python-version }}
|
|
23
|
+
cache: "pip"
|
|
24
|
+
|
|
25
|
+
# Install without extras so the suite proves the default install
|
|
26
|
+
# imports and works without optional SDKs such as google-genai.
|
|
27
|
+
- name: Install dependencies
|
|
28
|
+
run: |
|
|
29
|
+
python -m pip install --upgrade pip
|
|
30
|
+
python -m pip install -e . pytest
|
|
31
|
+
|
|
32
|
+
- name: Resolve the built wheel with current PyJWT
|
|
33
|
+
run: |
|
|
34
|
+
python -m pip install build
|
|
35
|
+
python -m build --wheel
|
|
36
|
+
python -m pip install --dry-run --ignore-installed dist/*.whl 'PyJWT[crypto]>=2.15.1,<3'
|
|
37
|
+
|
|
38
|
+
- name: Resolve the built wheel with protobuf 6
|
|
39
|
+
run: |
|
|
40
|
+
wheel=$(ls dist/*.whl)
|
|
41
|
+
python -m pip install --dry-run --ignore-installed "$wheel" 'protobuf>=6.31,<7'
|
|
42
|
+
python -m pip install --dry-run --ignore-installed "${wheel}[gemini]" 'protobuf>=6.31,<7'
|
|
43
|
+
|
|
44
|
+
# Preprovision immutable official data instead of asking the NLTK
|
|
45
|
+
# downloader to traverse the runner's proxy. Do not disable pathsec.
|
|
46
|
+
- name: Check out pinned NLTK test resources
|
|
47
|
+
uses: actions/checkout@v4
|
|
48
|
+
with:
|
|
49
|
+
repository: nltk/nltk_data
|
|
50
|
+
ref: 550b6625bcef1f2abff2ff770a5a0d272c9c6b2a
|
|
51
|
+
path: .ci-nltk-data
|
|
52
|
+
persist-credentials: false
|
|
53
|
+
sparse-checkout-cone-mode: false
|
|
54
|
+
sparse-checkout: |
|
|
55
|
+
packages/tokenizers/punkt.zip
|
|
56
|
+
packages/tokenizers/punkt_tab.zip
|
|
57
|
+
packages/corpora/wordnet.zip
|
|
58
|
+
packages/taggers/averaged_perceptron_tagger.zip
|
|
59
|
+
packages/taggers/averaged_perceptron_tagger_eng.zip
|
|
60
|
+
|
|
61
|
+
- name: Prepare and validate offline NLTK test resources
|
|
62
|
+
env:
|
|
63
|
+
DEEPDOC_NLTK_DATA_DIR: ${{ runner.temp }}/deepdoc-nltk-data
|
|
64
|
+
NLTK_DATA: ${{ runner.temp }}/deepdoc-nltk-data
|
|
65
|
+
run: |
|
|
66
|
+
for package in tokenizers/punkt tokenizers/punkt_tab corpora/wordnet \
|
|
67
|
+
taggers/averaged_perceptron_tagger taggers/averaged_perceptron_tagger_eng; do
|
|
68
|
+
python -m zipfile -e ".ci-nltk-data/packages/$package.zip" "$NLTK_DATA/$(dirname "$package")"
|
|
69
|
+
done
|
|
70
|
+
python -c 'from deepdoc.depend.nltk_manager import ensure_nltk_data; ensure_nltk_data(offline=True)'
|
|
71
|
+
|
|
72
|
+
- name: Run tests
|
|
73
|
+
env:
|
|
74
|
+
DEEPDOC_NLTK_DATA_DIR: ${{ runner.temp }}/deepdoc-nltk-data
|
|
75
|
+
NLTK_DATA: ${{ runner.temp }}/deepdoc-nltk-data
|
|
76
|
+
run: python -m pytest tests
|
|
77
|
+
|
|
78
|
+
- name: Test Gemini with the latest google-genai
|
|
79
|
+
run: |
|
|
80
|
+
python -m pip install -e ".[gemini]"
|
|
81
|
+
python -m pytest tests/test_gemini_sdk_compatibility.py
|
|
82
|
+
|
|
83
|
+
minimum-sdk:
|
|
84
|
+
name: Minimum SDK contract (3.10)
|
|
85
|
+
runs-on: ubuntu-latest
|
|
86
|
+
steps:
|
|
87
|
+
- uses: actions/checkout@v4
|
|
88
|
+
|
|
89
|
+
- name: Set up Python
|
|
90
|
+
uses: actions/setup-python@v5
|
|
91
|
+
with:
|
|
92
|
+
python-version: "3.10"
|
|
93
|
+
cache: "pip"
|
|
94
|
+
|
|
95
|
+
- name: Build and resolve the wheel with the minimum SDKs and current PyJWT
|
|
96
|
+
run: |
|
|
97
|
+
python -m pip install --upgrade pip build
|
|
98
|
+
python -m build --wheel
|
|
99
|
+
python -m pip install --dry-run --ignore-installed dist/*.whl 'zai-sdk==0.0.4' 'google-genai==1.0.0' 'PyJWT[crypto]>=2.15.1,<3'
|
|
100
|
+
|
|
101
|
+
- name: Install and test the minimum SDK contracts
|
|
102
|
+
run: |
|
|
103
|
+
python -m pip install dist/*.whl 'zai-sdk==0.0.4' 'google-genai==1.0.0' 'PyJWT[crypto]>=2.15.1,<3' pytest
|
|
104
|
+
python -m pip check
|
|
105
|
+
python -m pytest tests/test_zhipu_sdk_compatibility.py tests/test_gemini_sdk_compatibility.py
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: deepdoc_lib
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.5
|
|
4
4
|
Summary: [RAGFlow](https://ragflow.io/) is an open-source RAG (Retrieval-Augmented Generation) engine based on deep document understanding. It offers a streamlined RAG workflow for businesses of any scale, combining LLM (Large Language Models) to provide truthful question-answering capabilities, backed by well-founded citations from various complex formatted data.
|
|
5
5
|
Author-email: Zhichang Yu <yuzhichang@gmail.com>
|
|
6
6
|
Requires-Python: <3.14,>=3.10
|
|
@@ -42,14 +42,15 @@ Requires-Dist: tencentcloud-sdk-python>=3.0.1215
|
|
|
42
42
|
Requires-Dist: tiktoken>=0.7.0
|
|
43
43
|
Requires-Dist: xgboost<3.0.0,>=2.1.0
|
|
44
44
|
Requires-Dist: xpinyin>=0.7.6
|
|
45
|
-
Requires-Dist:
|
|
46
|
-
Requires-Dist: google-generativeai<0.9.0,>=0.8.1
|
|
45
|
+
Requires-Dist: zai-sdk>=0.0.4
|
|
47
46
|
Requires-Dist: trio>=0.29.0
|
|
48
47
|
Requires-Dist: setuptools<76.0.0,>=75.2.0
|
|
49
48
|
Requires-Dist: huggingface-hub<1.0.0,>=0.25.0
|
|
50
49
|
Provides-Extra: gpu
|
|
51
50
|
Requires-Dist: onnxruntime-gpu>=1.19.2; (sys_platform != "darwin" and platform_machine == "x86_64") and extra == "gpu"
|
|
52
51
|
Requires-Dist: torch; extra == "gpu"
|
|
52
|
+
Provides-Extra: gemini
|
|
53
|
+
Requires-Dist: google-genai>=1.0.0; extra == "gemini"
|
|
53
54
|
Dynamic: license-file
|
|
54
55
|
|
|
55
56
|
# Deepdoc
|
|
@@ -68,6 +69,12 @@ GPU (Linux x86_64 only):
|
|
|
68
69
|
pip install deepdoc-lib[gpu]
|
|
69
70
|
```
|
|
70
71
|
|
|
72
|
+
Gemini vision provider (optional, installs the `google-genai` SDK):
|
|
73
|
+
|
|
74
|
+
```bash
|
|
75
|
+
pip install deepdoc-lib[gemini]
|
|
76
|
+
```
|
|
77
|
+
|
|
71
78
|
Note: `onnxruntime` (CPU) and `onnxruntime-gpu` should not be installed together. If you're switching an existing environment to GPU, uninstall CPU ORT first:
|
|
72
79
|
|
|
73
80
|
```bash
|
|
@@ -222,6 +229,8 @@ vision_model = create_vision_model("qwen")
|
|
|
222
229
|
|
|
223
230
|
Supported providers: ["openai", "qwen", "zhipu", "ollama", "gemini", "anthropic"]
|
|
224
231
|
|
|
232
|
+
The `gemini` provider needs the `gemini` extra: `pip install deepdoc-lib[gemini]`.
|
|
233
|
+
|
|
225
234
|
- Use Configuration File
|
|
226
235
|
|
|
227
236
|
Create `deepdoc_config.yaml`:
|
|
@@ -14,6 +14,12 @@ GPU (Linux x86_64 only):
|
|
|
14
14
|
pip install deepdoc-lib[gpu]
|
|
15
15
|
```
|
|
16
16
|
|
|
17
|
+
Gemini vision provider (optional, installs the `google-genai` SDK):
|
|
18
|
+
|
|
19
|
+
```bash
|
|
20
|
+
pip install deepdoc-lib[gemini]
|
|
21
|
+
```
|
|
22
|
+
|
|
17
23
|
Note: `onnxruntime` (CPU) and `onnxruntime-gpu` should not be installed together. If you're switching an existing environment to GPU, uninstall CPU ORT first:
|
|
18
24
|
|
|
19
25
|
```bash
|
|
@@ -168,6 +174,8 @@ vision_model = create_vision_model("qwen")
|
|
|
168
174
|
|
|
169
175
|
Supported providers: ["openai", "qwen", "zhipu", "ollama", "gemini", "anthropic"]
|
|
170
176
|
|
|
177
|
+
The `gemini` provider needs the `gemini` extra: `pip install deepdoc-lib[gemini]`.
|
|
178
|
+
|
|
171
179
|
- Use Configuration File
|
|
172
180
|
|
|
173
181
|
Create `deepdoc_config.yaml`:
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
# Upstream Sync Record
|
|
2
|
+
|
|
3
|
+
This library is a standalone, pip-installable extraction of the `deepdoc/`
|
|
4
|
+
module from [infiniflow/ragflow](https://github.com/infiniflow/ragflow),
|
|
5
|
+
restructured around explicit configuration injection
|
|
6
|
+
(`PdfModelConfig` / `TokenizerConfig`, `DEEPDOC_*` env vars) and
|
|
7
|
+
ModelScope-based model distribution instead of ragflow's global settings and
|
|
8
|
+
HuggingFace runtime downloads.
|
|
9
|
+
|
|
10
|
+
Upstream changes are **ported by behavior, never by wholesale file copy**,
|
|
11
|
+
so the library's public API (`from deepdoc import PdfParser, ...`) and
|
|
12
|
+
packaging stay stable.
|
|
13
|
+
|
|
14
|
+
## Sync history
|
|
15
|
+
|
|
16
|
+
### 2026-07-23 — synced to ragflow `18ea0fb7` (2026-07-22)
|
|
17
|
+
|
|
18
|
+
Ported fixes (upstream PR references):
|
|
19
|
+
|
|
20
|
+
| Area | Fix | Upstream |
|
|
21
|
+
|---|---|---|
|
|
22
|
+
| vision/operators | `StandardizeImag` → `StandardizeImage` rename; the dynamic `getattr(operators, "StandardizeImage")` dispatch silently skipped image standardization before | [#16785](https://github.com/infiniflow/ragflow/pull/16785) |
|
|
23
|
+
| mineru/tcadp | zip extraction hardening: reject symlink / encrypted / absolute-path / `..`-traversal / zip-slip entries; streaming download for TCADP results | [#12527](https://github.com/infiniflow/ragflow/pull/12527) |
|
|
24
|
+
| html_parser | parse bodyless HTML fragments (`soup.body or soup`) | [#16423](https://github.com/infiniflow/ragflow/pull/16423) |
|
|
25
|
+
| html_parser | switch `html5lib` → `html.parser` (aligns with upstream #14486-era behavior the later fixes assume) | [#14486](https://github.com/infiniflow/ragflow/pull/14486) |
|
|
26
|
+
| html_parser | h4 heading mapping `#####` → `####` | [#13833](https://github.com/infiniflow/ragflow/pull/13833) |
|
|
27
|
+
| html_parser | oversized-block splitting preserves original text (CJK-aware atom splitting) | [#16052](https://github.com/infiniflow/ragflow/pull/16052) |
|
|
28
|
+
| excel_parser | keep `0` / `False` cells; skip fully-empty rows | [#16287](https://github.com/infiniflow/ragflow/pull/16287) |
|
|
29
|
+
| excel_parser | no spurious header-only chunk when data rows are an exact multiple of `chunk_rows` | [#15490](https://github.com/infiniflow/ragflow/pull/15490) |
|
|
30
|
+
| excel_parser | binary-search actual row count for sheets with abnormal `max_row` | [#13018](https://github.com/infiniflow/ragflow/pull/13018) |
|
|
31
|
+
| markdown_parser | full sync to upstream state: protected ranges (fenced code / MD tables / HTML tables) never split by delimiters; GFM separators with 1+ dashes; lone-header attachment | #10896 #11018 #11520 #13892 #15630 #15632 #16109 #16319 |
|
|
32
|
+
| ppt_parser | cached, None-safe shape sorting (group shapes with `top=None` no longer crash) | [#13054](https://github.com/infiniflow/ragflow/pull/13054) |
|
|
33
|
+
| parser/utils | `get_text`: `if binary:` → `if binary is not None:` | [#13196](https://github.com/infiniflow/ragflow/pull/13196) |
|
|
34
|
+
| table_structure_recognizer | English caption patterns (`Figure N` / `Table N` / `Fig. N`) | [#15481](https://github.com/infiniflow/ragflow/pull/15481) |
|
|
35
|
+
| vision/ocr | env-configurable ONNX thread counts (`OCR_INTRA_OP_NUM_THREADS` / `OCR_INTER_OP_NUM_THREADS`) and opt-in GPU arena shrinkage (`OCR_GPUMEM_ARENA_SHRINKAGE=1`) | [#12777](https://github.com/infiniflow/ragflow/pull/12777) |
|
|
36
|
+
| layout_recognizer | figure/equation placeholder boxes get per-type layoutno namespaces (`figure-N` / `equation-N`), preventing unrelated figure and equation crops from merging on scientific PDFs | [#15873](https://github.com/infiniflow/ragflow/pull/15873) |
|
|
37
|
+
| layout_recognizer | re-enable CID-pattern garbage filter for text boxes | [#13404](https://github.com/infiniflow/ragflow/pull/13404) |
|
|
38
|
+
| pdf_parser | garbled-text detection → OCR fallback (PUA/CID chars, subset-font encoding garbling, OCR-alphabet coverage check) | [#13404](https://github.com/infiniflow/ragflow/pull/13404) |
|
|
39
|
+
| pdf_parser | geometry-based word-space recovery, CJK-aware (`_insert_word_spaces`) | [#16958](https://github.com/infiniflow/ragflow/pull/16958) |
|
|
40
|
+
| pdf_parser | remove 299-page hardcoded limit (`MAXIMUM_PAGE_NUMBER = 100000`) | [#14382](https://github.com/infiniflow/ragflow/pull/14382) |
|
|
41
|
+
| pdf_parser | chunked `parse_into_bboxes` for large PDFs (`PDF_PARSER_PAGE_BATCH_SIZE`, default 50) with window→global box remapping; lazy `img_np` in `__ocr` | [#14385](https://github.com/infiniflow/ragflow/pull/14385) |
|
|
42
|
+
| pdf_parser | out-of-range page-index guards in `cropout` / table-figure insertion | [#12938](https://github.com/infiniflow/ragflow/pull/12938) [#12848](https://github.com/infiniflow/ragflow/pull/12848) |
|
|
43
|
+
| pdf_parser (VisionParser) | `pdf_page_num = from_page + idx` (page numbering was wrong when `from_page > 0`); fixed broken relative import of `llm_adapter` | [#12938](https://github.com/infiniflow/ragflow/pull/12938) + fork-local fix |
|
|
44
|
+
| parser (new) | EPUB parser (spine-order XHTML extraction, delegates to HtmlParser), exported as `deepdoc.EpubParser` | [#13650](https://github.com/infiniflow/ragflow/pull/13650) |
|
|
45
|
+
|
|
46
|
+
Already present before this sync (no action needed): `ast.literal_eval`
|
|
47
|
+
security fix (#12236), XLS/calamine loading (#10660), chartsheet guards
|
|
48
|
+
(#10819), KMeans multi-column detection (#11415 + #12534 cleanup),
|
|
49
|
+
`is_english` regex fix (#11432), OCR GPU memory limit env (#10407),
|
|
50
|
+
`vision_figure_parser_docx_wrapper` guard (#12104).
|
|
51
|
+
|
|
52
|
+
**Deliberately NOT ported** (decisions):
|
|
53
|
+
|
|
54
|
+
- **Go + HTTP microservice pipeline** (#16323, `deepdoc/server/`, `DEEPDOC_URL`
|
|
55
|
+
DLA client): conflicts with this library's pure-Python, in-process design.
|
|
56
|
+
- **Automatic table orientation detection/rotation** (#12719 feature +
|
|
57
|
+
#12981 / #17016 follow-ups): feature not in our baseline; runs 4× OCR per
|
|
58
|
+
table; adopt separately if ever needed.
|
|
59
|
+
- **`rag.utils.lazy_image` memory refactors** (docx/excel lazy image loading,
|
|
60
|
+
#13329 / #13558): optimization only; deferred until vendoring lazy_image is
|
|
61
|
+
justified.
|
|
62
|
+
- **3-tuple outlines** (`extract_pdf_outlines` returning
|
|
63
|
+
`(title, depth, page)`, #10456 TOC feature): our `outlines` stays
|
|
64
|
+
`(title, depth)` to avoid breaking downstream consumers.
|
|
65
|
+
- **`model_speciess` → `model_species` rename** (#13929): our attribute name
|
|
66
|
+
is part of the public surface; renaming would break existing callers.
|
|
67
|
+
- **paddleocr / opendataloader / somark parsers**: cloud-API clients, not
|
|
68
|
+
currently needed; revisit on demand.
|
|
69
|
+
- **MinerU feature evolution** (API-only backends, VLM descriptions, etc.):
|
|
70
|
+
only the security fix was taken; feature sync deferred until MinerU usage
|
|
71
|
+
is confirmed.
|
|
72
|
+
- **Pure refactors** (moving functions between ragflow modules, lefthook
|
|
73
|
+
reformatting): not applicable to our layout.
|
|
74
|
+
|
|
75
|
+
Regression tests: `tests/test_upstream_sync_regressions.py`.
|
|
76
|
+
|
|
77
|
+
## How to run the next sync
|
|
78
|
+
|
|
79
|
+
```bash
|
|
80
|
+
# in a ragflow clone
|
|
81
|
+
git log --reverse --format="%h %ad %s" --date=short 18ea0fb7..origin/main -- deepdoc
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
Then for each commit: understand the behavior change, find the corresponding
|
|
85
|
+
code in this repo, and re-implement it in this library's style (config
|
|
86
|
+
injection, relative imports, vendored `common/` + `depend/`). Update this file.
|
|
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
|
|
|
18
18
|
commit_id: str | None
|
|
19
19
|
__commit_id__: str | None
|
|
20
20
|
|
|
21
|
-
__version__ = version = '0.2.
|
|
22
|
-
__version_tuple__ = version_tuple = (0, 2,
|
|
21
|
+
__version__ = version = '0.2.5'
|
|
22
|
+
__version_tuple__ = version_tuple = (0, 2, 5)
|
|
23
23
|
|
|
24
|
-
__commit_id__ = commit_id = '
|
|
24
|
+
__commit_id__ = commit_id = 'g8ef70a65b'
|
|
@@ -11,7 +11,7 @@ import requests
|
|
|
11
11
|
from openai import OpenAI
|
|
12
12
|
from openai.lib.azure import AzureOpenAI
|
|
13
13
|
from PIL import Image
|
|
14
|
-
from
|
|
14
|
+
from zai import ZhipuAiClient
|
|
15
15
|
|
|
16
16
|
# 可选导入 ollama
|
|
17
17
|
try:
|
|
@@ -162,7 +162,12 @@ class Zhipu4V(Base):
|
|
|
162
162
|
|
|
163
163
|
def __init__(self, key, model_name="glm-4v", lang="Chinese", **kwargs):
|
|
164
164
|
super().__init__(key, model_name, lang)
|
|
165
|
-
|
|
165
|
+
# Preserve the legacy override and China default instead of allowing
|
|
166
|
+
# the maintained SDK's ZAI_BASE_URL to redirect this provider adapter.
|
|
167
|
+
self.client = ZhipuAiClient(
|
|
168
|
+
api_key=key,
|
|
169
|
+
base_url=kwargs.get("base_url") or os.getenv("ZHIPUAI_BASE_URL") or "https://open.bigmodel.cn/api/paas/v4/",
|
|
170
|
+
)
|
|
166
171
|
|
|
167
172
|
def describe_with_prompt(self, image, prompt=None):
|
|
168
173
|
"""使用智谱AI视觉模型描述图片"""
|
|
@@ -301,9 +306,16 @@ class GeminiCV(Base):
|
|
|
301
306
|
|
|
302
307
|
def __init__(self, key, model_name="gemini-pro-vision", lang="Chinese", **kwargs):
|
|
303
308
|
super().__init__(key, model_name, lang)
|
|
304
|
-
|
|
305
|
-
genai.
|
|
306
|
-
|
|
309
|
+
# The legacy google-generativeai SDK caps protobuf below 6, so Gemini
|
|
310
|
+
# uses the maintained google-genai SDK as an optional extra instead.
|
|
311
|
+
try:
|
|
312
|
+
from google import genai
|
|
313
|
+
except ImportError as e:
|
|
314
|
+
raise ImportError(
|
|
315
|
+
"Gemini vision requires the google-genai package. "
|
|
316
|
+
"Install it with: pip install 'deepdoc-lib[gemini]'"
|
|
317
|
+
) from e
|
|
318
|
+
self.client = genai.Client(api_key=key)
|
|
307
319
|
|
|
308
320
|
def describe_with_prompt(self, image, prompt=None):
|
|
309
321
|
"""使用Google Gemini模型描述图片"""
|
|
@@ -318,11 +330,19 @@ class GeminiCV(Base):
|
|
|
318
330
|
|
|
319
331
|
pil_image = Image.open(BytesIO(img_data))
|
|
320
332
|
|
|
321
|
-
response = self.
|
|
322
|
-
|
|
323
|
-
pil_image
|
|
324
|
-
|
|
333
|
+
response = self.client.models.generate_content(
|
|
334
|
+
model=self.model_name,
|
|
335
|
+
contents=[prompt or self.prompt(image), pil_image],
|
|
336
|
+
)
|
|
325
337
|
|
|
338
|
+
# google-genai returns None (the legacy SDK raised) when the
|
|
339
|
+
# response has no text, e.g. a safety block; keep the error path.
|
|
340
|
+
# A block can be reported on the prompt or on the candidate.
|
|
341
|
+
if response.text is None:
|
|
342
|
+
candidates = response.candidates or []
|
|
343
|
+
finish_reason = candidates[0].finish_reason if candidates else None
|
|
344
|
+
finish_reason = getattr(finish_reason, "name", finish_reason)
|
|
345
|
+
raise ValueError(f"Gemini returned no text (finish_reason={finish_reason}, prompt_feedback={response.prompt_feedback})")
|
|
326
346
|
return response.text.strip()
|
|
327
347
|
except Exception as e:
|
|
328
348
|
return "**ERROR**: " + str(e)
|
|
@@ -466,4 +486,4 @@ def create_vision_model(config=None):
|
|
|
466
486
|
return VisionModelFactory.create_model(config)
|
|
467
487
|
|
|
468
488
|
else:
|
|
469
|
-
raise ValueError(f"不支持的配置类型: {type(config)}")
|
|
489
|
+
raise ValueError(f"不支持的配置类型: {type(config)}")
|
|
@@ -16,6 +16,7 @@
|
|
|
16
16
|
|
|
17
17
|
from .docx_parser import RAGFlowDocxParser as DocxParser
|
|
18
18
|
from .docling_parser import DoclingParser
|
|
19
|
+
from .epub_parser import RAGFlowEpubParser as EpubParser
|
|
19
20
|
from .excel_parser import RAGFlowExcelParser as ExcelParser
|
|
20
21
|
from .html_parser import RAGFlowHtmlParser as HtmlParser
|
|
21
22
|
from .json_parser import RAGFlowJsonParser as JsonParser
|
|
@@ -31,6 +32,7 @@ __all__ = [
|
|
|
31
32
|
"PlainParser",
|
|
32
33
|
"DocxParser",
|
|
33
34
|
"DoclingParser",
|
|
35
|
+
"EpubParser",
|
|
34
36
|
"ExcelParser",
|
|
35
37
|
"PptParser",
|
|
36
38
|
"HtmlParser",
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
|
|
3
|
+
#
|
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
# you may not use this file except in compliance with the License.
|
|
6
|
+
# You may obtain a copy of the License at
|
|
7
|
+
#
|
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
#
|
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
# See the License for the specific language governing permissions and
|
|
14
|
+
# limitations under the License.
|
|
15
|
+
#
|
|
16
|
+
|
|
17
|
+
import logging
|
|
18
|
+
import warnings
|
|
19
|
+
import zipfile
|
|
20
|
+
from io import BytesIO
|
|
21
|
+
from xml.etree import ElementTree
|
|
22
|
+
|
|
23
|
+
from ..config import TokenizerConfig
|
|
24
|
+
from .html_parser import RAGFlowHtmlParser
|
|
25
|
+
|
|
26
|
+
# OPF XML namespaces
|
|
27
|
+
_OPF_NS = "http://www.idpf.org/2007/opf"
|
|
28
|
+
_CONTAINER_NS = "urn:oasis:names:tc:opendocument:xmlns:container"
|
|
29
|
+
|
|
30
|
+
# Media types that contain readable XHTML content
|
|
31
|
+
_XHTML_MEDIA_TYPES = {"application/xhtml+xml", "text/html", "text/xml"}
|
|
32
|
+
|
|
33
|
+
logger = logging.getLogger(__name__)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class RAGFlowEpubParser:
|
|
37
|
+
"""Parse EPUB files by extracting XHTML content in spine (reading) order
|
|
38
|
+
and delegating to RAGFlowHtmlParser for chunking."""
|
|
39
|
+
|
|
40
|
+
def __init__(self, tokenizer_cfg: TokenizerConfig | None = None):
|
|
41
|
+
self.html_parser = RAGFlowHtmlParser(tokenizer_cfg=tokenizer_cfg)
|
|
42
|
+
|
|
43
|
+
def __call__(self, fnm, binary=None, chunk_token_num=512):
|
|
44
|
+
if binary is not None:
|
|
45
|
+
if not binary:
|
|
46
|
+
logger.warning(
|
|
47
|
+
"RAGFlowEpubParser received an empty EPUB binary payload for %r",
|
|
48
|
+
fnm,
|
|
49
|
+
)
|
|
50
|
+
raise ValueError("Empty EPUB binary payload")
|
|
51
|
+
zf = zipfile.ZipFile(BytesIO(binary))
|
|
52
|
+
else:
|
|
53
|
+
zf = zipfile.ZipFile(fnm)
|
|
54
|
+
|
|
55
|
+
try:
|
|
56
|
+
content_items = self._get_spine_items(zf)
|
|
57
|
+
all_sections = []
|
|
58
|
+
|
|
59
|
+
for item_path in content_items:
|
|
60
|
+
try:
|
|
61
|
+
html_bytes = zf.read(item_path)
|
|
62
|
+
except KeyError:
|
|
63
|
+
continue
|
|
64
|
+
if not html_bytes:
|
|
65
|
+
logger.debug("Skipping empty EPUB content item: %s", item_path)
|
|
66
|
+
continue
|
|
67
|
+
with warnings.catch_warnings():
|
|
68
|
+
warnings.filterwarnings("ignore", category=UserWarning)
|
|
69
|
+
sections = self.html_parser(item_path, binary=html_bytes, chunk_token_num=chunk_token_num)
|
|
70
|
+
all_sections.extend(sections)
|
|
71
|
+
|
|
72
|
+
return all_sections
|
|
73
|
+
finally:
|
|
74
|
+
zf.close()
|
|
75
|
+
|
|
76
|
+
@staticmethod
|
|
77
|
+
def _get_spine_items(zf):
|
|
78
|
+
"""Return content file paths in spine (reading) order."""
|
|
79
|
+
# 1. Find the OPF file path from META-INF/container.xml
|
|
80
|
+
try:
|
|
81
|
+
container_xml = zf.read("META-INF/container.xml")
|
|
82
|
+
except KeyError:
|
|
83
|
+
return RAGFlowEpubParser._fallback_xhtml_order(zf)
|
|
84
|
+
|
|
85
|
+
try:
|
|
86
|
+
container_root = ElementTree.fromstring(container_xml)
|
|
87
|
+
except ElementTree.ParseError:
|
|
88
|
+
logger.warning("Failed to parse META-INF/container.xml; falling back to XHTML order.")
|
|
89
|
+
return RAGFlowEpubParser._fallback_xhtml_order(zf)
|
|
90
|
+
|
|
91
|
+
rootfile_el = container_root.find(f".//{{{_CONTAINER_NS}}}rootfile")
|
|
92
|
+
if rootfile_el is None:
|
|
93
|
+
return RAGFlowEpubParser._fallback_xhtml_order(zf)
|
|
94
|
+
|
|
95
|
+
opf_path = rootfile_el.get("full-path", "")
|
|
96
|
+
if not opf_path:
|
|
97
|
+
return RAGFlowEpubParser._fallback_xhtml_order(zf)
|
|
98
|
+
|
|
99
|
+
# Base directory of the OPF file (content paths are relative to it)
|
|
100
|
+
opf_dir = opf_path.rsplit("/", 1)[0] + "/" if "/" in opf_path else ""
|
|
101
|
+
|
|
102
|
+
# 2. Parse the OPF file
|
|
103
|
+
try:
|
|
104
|
+
opf_xml = zf.read(opf_path)
|
|
105
|
+
except KeyError:
|
|
106
|
+
return RAGFlowEpubParser._fallback_xhtml_order(zf)
|
|
107
|
+
|
|
108
|
+
try:
|
|
109
|
+
opf_root = ElementTree.fromstring(opf_xml)
|
|
110
|
+
except ElementTree.ParseError:
|
|
111
|
+
logger.warning("Failed to parse OPF file '%s'; falling back to XHTML order.", opf_path)
|
|
112
|
+
return RAGFlowEpubParser._fallback_xhtml_order(zf)
|
|
113
|
+
|
|
114
|
+
# 3. Build id->href+mediatype map from <manifest>
|
|
115
|
+
manifest = {}
|
|
116
|
+
for item in opf_root.findall(f".//{{{_OPF_NS}}}item"):
|
|
117
|
+
item_id = item.get("id", "")
|
|
118
|
+
href = item.get("href", "")
|
|
119
|
+
media_type = item.get("media-type", "")
|
|
120
|
+
if item_id and href:
|
|
121
|
+
manifest[item_id] = (href, media_type)
|
|
122
|
+
|
|
123
|
+
# 4. Walk <spine> to get reading order
|
|
124
|
+
spine_items = []
|
|
125
|
+
for itemref in opf_root.findall(f".//{{{_OPF_NS}}}itemref"):
|
|
126
|
+
idref = itemref.get("idref", "")
|
|
127
|
+
if idref not in manifest:
|
|
128
|
+
continue
|
|
129
|
+
href, media_type = manifest[idref]
|
|
130
|
+
if media_type not in _XHTML_MEDIA_TYPES:
|
|
131
|
+
continue
|
|
132
|
+
spine_items.append(opf_dir + href)
|
|
133
|
+
|
|
134
|
+
return spine_items if spine_items else RAGFlowEpubParser._fallback_xhtml_order(zf)
|
|
135
|
+
|
|
136
|
+
@staticmethod
|
|
137
|
+
def _fallback_xhtml_order(zf):
|
|
138
|
+
"""Fallback: return all .xhtml/.html files sorted alphabetically."""
|
|
139
|
+
return sorted(n for n in zf.namelist() if n.lower().endswith((".xhtml", ".html", ".htm")) and not n.startswith("META-INF/"))
|