deepdoc-lib 0.2.2__tar.gz → 0.2.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/.env.example +1 -1
  2. deepdoc_lib-0.2.5/.github/workflows/ci.yml +105 -0
  3. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/PKG-INFO +12 -3
  4. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/README.md +8 -0
  5. deepdoc_lib-0.2.5/UPSTREAM_SYNC.md +86 -0
  6. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/__init__.py +1 -0
  7. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/_version.py +3 -3
  8. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/depend/simple_cv_model.py +30 -10
  9. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/__init__.py +2 -0
  10. deepdoc_lib-0.2.5/deepdoc/parser/epub_parser.py +139 -0
  11. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/excel_parser.py +78 -25
  12. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/html_parser.py +77 -14
  13. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/markdown_parser.py +240 -34
  14. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/mineru_parser.py +42 -22
  15. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/pdf_parser.py +388 -41
  16. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/ppt_parser.py +9 -3
  17. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/tcadp_parser.py +43 -23
  18. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/utils.py +1 -1
  19. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/vision/layout_recognizer.py +35 -24
  20. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/vision/ocr.py +8 -2
  21. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/vision/operators.py +1 -1
  22. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/vision/table_structure_recognizer.py +6 -1
  23. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc_lib.egg-info/PKG-INFO +12 -3
  24. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc_lib.egg-info/SOURCES.txt +2 -0
  25. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc_lib.egg-info/requires.txt +4 -2
  26. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc_lib.egg-info/scm_file_list.json +72 -67
  27. deepdoc_lib-0.2.5/deepdoc_lib.egg-info/scm_version.json +8 -0
  28. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/pyproject.toml +2 -2
  29. deepdoc_lib-0.2.2/.github/workflows/ci.yml +0 -31
  30. deepdoc_lib-0.2.2/deepdoc_lib.egg-info/scm_version.json +0 -8
  31. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/.github/workflows/publish.yml +0 -0
  32. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/LICENSE +0 -0
  33. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/MANIFEST.in +0 -0
  34. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/README.md +0 -0
  35. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/README_zh.md +0 -0
  36. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/common/__init__.py +0 -0
  37. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/common/config_utils.py +0 -0
  38. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/common/connection_utils.py +0 -0
  39. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/common/file_utils.py +0 -0
  40. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/common/misc_utils.py +0 -0
  41. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/common/model_store.py +0 -0
  42. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/common/settings.py +0 -0
  43. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/common/tiktoken_cache.py +0 -0
  44. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/common/token_utils.py +0 -0
  45. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/config.py +0 -0
  46. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/depend/find_codec.py +0 -0
  47. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/depend/nltk_manager.py +0 -0
  48. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/depend/prompts/vision_llm_describe_prompt.md +0 -0
  49. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/depend/prompts/vision_llm_figure_describe_prompt.md +0 -0
  50. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/depend/prompts.py +0 -0
  51. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/depend/rag_tokenizer.py +0 -0
  52. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/depend/surname.py +0 -0
  53. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/depend/timeout.py +0 -0
  54. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/depend/vision_llm_chunk.py +0 -0
  55. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/dict/README.md +0 -0
  56. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/dict/huqie.txt +0 -0
  57. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/download_models.py +0 -0
  58. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/llm_adapter/__init__.py +0 -0
  59. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/llm_adapter/adapter.py +0 -0
  60. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/llm_adapter/utils.py +0 -0
  61. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/llm_adapter/vision.py +0 -0
  62. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/docling_parser.py +0 -0
  63. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/docx_parser.py +0 -0
  64. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/figure_parser.py +0 -0
  65. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/json_parser.py +0 -0
  66. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/resume/__init__.py +0 -0
  67. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/resume/entities/__init__.py +0 -0
  68. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/resume/entities/corporations.py +0 -0
  69. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/resume/entities/degrees.py +0 -0
  70. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/resume/entities/industries.py +0 -0
  71. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/resume/entities/regions.py +0 -0
  72. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/resume/entities/res/corp.tks.freq.json +0 -0
  73. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/resume/entities/res/corp_baike_len.csv +0 -0
  74. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/resume/entities/res/corp_tag.json +0 -0
  75. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/resume/entities/res/good_corp.json +0 -0
  76. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/resume/entities/res/good_sch.json +0 -0
  77. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/resume/entities/res/school.rank.csv +0 -0
  78. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/resume/entities/res/schools.csv +0 -0
  79. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/resume/entities/schools.py +0 -0
  80. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/resume/step_one.py +0 -0
  81. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/resume/step_two.py +0 -0
  82. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/parser/txt_parser.py +0 -0
  83. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/vision/__init__.py +0 -0
  84. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/vision/postprocess.py +0 -0
  85. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/vision/recognizer.py +0 -0
  86. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/vision/seeit.py +0 -0
  87. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/vision/t_ocr.py +0 -0
  88. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc/vision/t_recognizer.py +0 -0
  89. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc_config.example.yaml +0 -0
  90. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc_lib.egg-info/dependency_links.txt +0 -0
  91. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc_lib.egg-info/entry_points.txt +0 -0
  92. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/deepdoc_lib.egg-info/top_level.txt +0 -0
  93. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/scripts/download_models.py +0 -0
  94. {deepdoc_lib-0.2.2 → deepdoc_lib-0.2.5}/setup.cfg +0 -0
@@ -29,7 +29,7 @@ DEEPDOC_VISION_BASE_URL=
29
29
  # openai - OpenAI GPT-4 Vision
30
30
  # qwen - 阿里云通义千问
31
31
  # zhipu - 智谱AI
32
- # gemini - Google Gemini
32
+ # gemini - Google Gemini (需安装: pip install deepdoc-lib[gemini])
33
33
  # ollama - 本地 Ollama 服务
34
34
  # anthropic - Anthropic Claude (需额外配置)
35
35
  DEEPDOC_VISION_PROVIDER=qwen
@@ -0,0 +1,105 @@
1
+ name: Tests
2
+
3
+ on:
4
+ push:
5
+ pull_request:
6
+ workflow_dispatch:
7
+
8
+ jobs:
9
+ pytest:
10
+ name: Pytest (${{ matrix.python-version }})
11
+ runs-on: ubuntu-latest
12
+ strategy:
13
+ matrix:
14
+ python-version: ["3.10", "3.11", "3.12", "3.13"]
15
+
16
+ steps:
17
+ - uses: actions/checkout@v4
18
+
19
+ - name: Set up Python
20
+ uses: actions/setup-python@v5
21
+ with:
22
+ python-version: ${{ matrix.python-version }}
23
+ cache: "pip"
24
+
25
+ # Install without extras so the suite proves the default install
26
+ # imports and works without optional SDKs such as google-genai.
27
+ - name: Install dependencies
28
+ run: |
29
+ python -m pip install --upgrade pip
30
+ python -m pip install -e . pytest
31
+
32
+ - name: Resolve the built wheel with current PyJWT
33
+ run: |
34
+ python -m pip install build
35
+ python -m build --wheel
36
+ python -m pip install --dry-run --ignore-installed dist/*.whl 'PyJWT[crypto]>=2.15.1,<3'
37
+
38
+ - name: Resolve the built wheel with protobuf 6
39
+ run: |
40
+ wheel=$(ls dist/*.whl)
41
+ python -m pip install --dry-run --ignore-installed "$wheel" 'protobuf>=6.31,<7'
42
+ python -m pip install --dry-run --ignore-installed "${wheel}[gemini]" 'protobuf>=6.31,<7'
43
+
44
+ # Preprovision immutable official data instead of asking the NLTK
45
+ # downloader to traverse the runner's proxy. Do not disable pathsec.
46
+ - name: Check out pinned NLTK test resources
47
+ uses: actions/checkout@v4
48
+ with:
49
+ repository: nltk/nltk_data
50
+ ref: 550b6625bcef1f2abff2ff770a5a0d272c9c6b2a
51
+ path: .ci-nltk-data
52
+ persist-credentials: false
53
+ sparse-checkout-cone-mode: false
54
+ sparse-checkout: |
55
+ packages/tokenizers/punkt.zip
56
+ packages/tokenizers/punkt_tab.zip
57
+ packages/corpora/wordnet.zip
58
+ packages/taggers/averaged_perceptron_tagger.zip
59
+ packages/taggers/averaged_perceptron_tagger_eng.zip
60
+
61
+ - name: Prepare and validate offline NLTK test resources
62
+ env:
63
+ DEEPDOC_NLTK_DATA_DIR: ${{ runner.temp }}/deepdoc-nltk-data
64
+ NLTK_DATA: ${{ runner.temp }}/deepdoc-nltk-data
65
+ run: |
66
+ for package in tokenizers/punkt tokenizers/punkt_tab corpora/wordnet \
67
+ taggers/averaged_perceptron_tagger taggers/averaged_perceptron_tagger_eng; do
68
+ python -m zipfile -e ".ci-nltk-data/packages/$package.zip" "$NLTK_DATA/$(dirname "$package")"
69
+ done
70
+ python -c 'from deepdoc.depend.nltk_manager import ensure_nltk_data; ensure_nltk_data(offline=True)'
71
+
72
+ - name: Run tests
73
+ env:
74
+ DEEPDOC_NLTK_DATA_DIR: ${{ runner.temp }}/deepdoc-nltk-data
75
+ NLTK_DATA: ${{ runner.temp }}/deepdoc-nltk-data
76
+ run: python -m pytest tests
77
+
78
+ - name: Test Gemini with the latest google-genai
79
+ run: |
80
+ python -m pip install -e ".[gemini]"
81
+ python -m pytest tests/test_gemini_sdk_compatibility.py
82
+
83
+ minimum-sdk:
84
+ name: Minimum SDK contract (3.10)
85
+ runs-on: ubuntu-latest
86
+ steps:
87
+ - uses: actions/checkout@v4
88
+
89
+ - name: Set up Python
90
+ uses: actions/setup-python@v5
91
+ with:
92
+ python-version: "3.10"
93
+ cache: "pip"
94
+
95
+ - name: Build and resolve the wheel with the minimum SDKs and current PyJWT
96
+ run: |
97
+ python -m pip install --upgrade pip build
98
+ python -m build --wheel
99
+ python -m pip install --dry-run --ignore-installed dist/*.whl 'zai-sdk==0.0.4' 'google-genai==1.0.0' 'PyJWT[crypto]>=2.15.1,<3'
100
+
101
+ - name: Install and test the minimum SDK contracts
102
+ run: |
103
+ python -m pip install dist/*.whl 'zai-sdk==0.0.4' 'google-genai==1.0.0' 'PyJWT[crypto]>=2.15.1,<3' pytest
104
+ python -m pip check
105
+ python -m pytest tests/test_zhipu_sdk_compatibility.py tests/test_gemini_sdk_compatibility.py
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: deepdoc_lib
3
- Version: 0.2.2
3
+ Version: 0.2.5
4
4
  Summary: [RAGFlow](https://ragflow.io/) is an open-source RAG (Retrieval-Augmented Generation) engine based on deep document understanding. It offers a streamlined RAG workflow for businesses of any scale, combining LLM (Large Language Models) to provide truthful question-answering capabilities, backed by well-founded citations from various complex formatted data.
5
5
  Author-email: Zhichang Yu <yuzhichang@gmail.com>
6
6
  Requires-Python: <3.14,>=3.10
@@ -42,14 +42,15 @@ Requires-Dist: tencentcloud-sdk-python>=3.0.1215
42
42
  Requires-Dist: tiktoken>=0.7.0
43
43
  Requires-Dist: xgboost<3.0.0,>=2.1.0
44
44
  Requires-Dist: xpinyin>=0.7.6
45
- Requires-Dist: zhipuai>=2.0.1
46
- Requires-Dist: google-generativeai<0.9.0,>=0.8.1
45
+ Requires-Dist: zai-sdk>=0.0.4
47
46
  Requires-Dist: trio>=0.29.0
48
47
  Requires-Dist: setuptools<76.0.0,>=75.2.0
49
48
  Requires-Dist: huggingface-hub<1.0.0,>=0.25.0
50
49
  Provides-Extra: gpu
51
50
  Requires-Dist: onnxruntime-gpu>=1.19.2; (sys_platform != "darwin" and platform_machine == "x86_64") and extra == "gpu"
52
51
  Requires-Dist: torch; extra == "gpu"
52
+ Provides-Extra: gemini
53
+ Requires-Dist: google-genai>=1.0.0; extra == "gemini"
53
54
  Dynamic: license-file
54
55
 
55
56
  # Deepdoc
@@ -68,6 +69,12 @@ GPU (Linux x86_64 only):
68
69
  pip install deepdoc-lib[gpu]
69
70
  ```
70
71
 
72
+ Gemini vision provider (optional, installs the `google-genai` SDK):
73
+
74
+ ```bash
75
+ pip install deepdoc-lib[gemini]
76
+ ```
77
+
71
78
  Note: `onnxruntime` (CPU) and `onnxruntime-gpu` should not be installed together. If you're switching an existing environment to GPU, uninstall CPU ORT first:
72
79
 
73
80
  ```bash
@@ -222,6 +229,8 @@ vision_model = create_vision_model("qwen")
222
229
 
223
230
  Supported providers: ["openai", "qwen", "zhipu", "ollama", "gemini", "anthropic"]
224
231
 
232
+ The `gemini` provider needs the `gemini` extra: `pip install deepdoc-lib[gemini]`.
233
+
225
234
  - Use Configuration File
226
235
 
227
236
  Create `deepdoc_config.yaml`:
@@ -14,6 +14,12 @@ GPU (Linux x86_64 only):
14
14
  pip install deepdoc-lib[gpu]
15
15
  ```
16
16
 
17
+ Gemini vision provider (optional, installs the `google-genai` SDK):
18
+
19
+ ```bash
20
+ pip install deepdoc-lib[gemini]
21
+ ```
22
+
17
23
  Note: `onnxruntime` (CPU) and `onnxruntime-gpu` should not be installed together. If you're switching an existing environment to GPU, uninstall CPU ORT first:
18
24
 
19
25
  ```bash
@@ -168,6 +174,8 @@ vision_model = create_vision_model("qwen")
168
174
 
169
175
  Supported providers: ["openai", "qwen", "zhipu", "ollama", "gemini", "anthropic"]
170
176
 
177
+ The `gemini` provider needs the `gemini` extra: `pip install deepdoc-lib[gemini]`.
178
+
171
179
  - Use Configuration File
172
180
 
173
181
  Create `deepdoc_config.yaml`:
@@ -0,0 +1,86 @@
1
+ # Upstream Sync Record
2
+
3
+ This library is a standalone, pip-installable extraction of the `deepdoc/`
4
+ module from [infiniflow/ragflow](https://github.com/infiniflow/ragflow),
5
+ restructured around explicit configuration injection
6
+ (`PdfModelConfig` / `TokenizerConfig`, `DEEPDOC_*` env vars) and
7
+ ModelScope-based model distribution instead of ragflow's global settings and
8
+ HuggingFace runtime downloads.
9
+
10
+ Upstream changes are **ported by behavior, never by wholesale file copy**,
11
+ so the library's public API (`from deepdoc import PdfParser, ...`) and
12
+ packaging stay stable.
13
+
14
+ ## Sync history
15
+
16
+ ### 2026-07-23 — synced to ragflow `18ea0fb7` (2026-07-22)
17
+
18
+ Ported fixes (upstream PR references):
19
+
20
+ | Area | Fix | Upstream |
21
+ |---|---|---|
22
+ | vision/operators | `StandardizeImag` → `StandardizeImage` rename; the dynamic `getattr(operators, "StandardizeImage")` dispatch silently skipped image standardization before | [#16785](https://github.com/infiniflow/ragflow/pull/16785) |
23
+ | mineru/tcadp | zip extraction hardening: reject symlink / encrypted / absolute-path / `..`-traversal / zip-slip entries; streaming download for TCADP results | [#12527](https://github.com/infiniflow/ragflow/pull/12527) |
24
+ | html_parser | parse bodyless HTML fragments (`soup.body or soup`) | [#16423](https://github.com/infiniflow/ragflow/pull/16423) |
25
+ | html_parser | switch `html5lib` → `html.parser` (aligns with upstream #14486-era behavior the later fixes assume) | [#14486](https://github.com/infiniflow/ragflow/pull/14486) |
26
+ | html_parser | h4 heading mapping `#####` → `####` | [#13833](https://github.com/infiniflow/ragflow/pull/13833) |
27
+ | html_parser | oversized-block splitting preserves original text (CJK-aware atom splitting) | [#16052](https://github.com/infiniflow/ragflow/pull/16052) |
28
+ | excel_parser | keep `0` / `False` cells; skip fully-empty rows | [#16287](https://github.com/infiniflow/ragflow/pull/16287) |
29
+ | excel_parser | no spurious header-only chunk when data rows are an exact multiple of `chunk_rows` | [#15490](https://github.com/infiniflow/ragflow/pull/15490) |
30
+ | excel_parser | binary-search actual row count for sheets with abnormal `max_row` | [#13018](https://github.com/infiniflow/ragflow/pull/13018) |
31
+ | markdown_parser | full sync to upstream state: protected ranges (fenced code / MD tables / HTML tables) never split by delimiters; GFM separators with 1+ dashes; lone-header attachment | #10896 #11018 #11520 #13892 #15630 #15632 #16109 #16319 |
32
+ | ppt_parser | cached, None-safe shape sorting (group shapes with `top=None` no longer crash) | [#13054](https://github.com/infiniflow/ragflow/pull/13054) |
33
+ | parser/utils | `get_text`: `if binary:` → `if binary is not None:` | [#13196](https://github.com/infiniflow/ragflow/pull/13196) |
34
+ | table_structure_recognizer | English caption patterns (`Figure N` / `Table N` / `Fig. N`) | [#15481](https://github.com/infiniflow/ragflow/pull/15481) |
35
+ | vision/ocr | env-configurable ONNX thread counts (`OCR_INTRA_OP_NUM_THREADS` / `OCR_INTER_OP_NUM_THREADS`) and opt-in GPU arena shrinkage (`OCR_GPUMEM_ARENA_SHRINKAGE=1`) | [#12777](https://github.com/infiniflow/ragflow/pull/12777) |
36
+ | layout_recognizer | figure/equation placeholder boxes get per-type layoutno namespaces (`figure-N` / `equation-N`), preventing unrelated figure and equation crops from merging on scientific PDFs | [#15873](https://github.com/infiniflow/ragflow/pull/15873) |
37
+ | layout_recognizer | re-enable CID-pattern garbage filter for text boxes | [#13404](https://github.com/infiniflow/ragflow/pull/13404) |
38
+ | pdf_parser | garbled-text detection → OCR fallback (PUA/CID chars, subset-font encoding garbling, OCR-alphabet coverage check) | [#13404](https://github.com/infiniflow/ragflow/pull/13404) |
39
+ | pdf_parser | geometry-based word-space recovery, CJK-aware (`_insert_word_spaces`) | [#16958](https://github.com/infiniflow/ragflow/pull/16958) |
40
+ | pdf_parser | remove 299-page hardcoded limit (`MAXIMUM_PAGE_NUMBER = 100000`) | [#14382](https://github.com/infiniflow/ragflow/pull/14382) |
41
+ | pdf_parser | chunked `parse_into_bboxes` for large PDFs (`PDF_PARSER_PAGE_BATCH_SIZE`, default 50) with window→global box remapping; lazy `img_np` in `__ocr` | [#14385](https://github.com/infiniflow/ragflow/pull/14385) |
42
+ | pdf_parser | out-of-range page-index guards in `cropout` / table-figure insertion | [#12938](https://github.com/infiniflow/ragflow/pull/12938) [#12848](https://github.com/infiniflow/ragflow/pull/12848) |
43
+ | pdf_parser (VisionParser) | `pdf_page_num = from_page + idx` (page numbering was wrong when `from_page > 0`); fixed broken relative import of `llm_adapter` | [#12938](https://github.com/infiniflow/ragflow/pull/12938) + fork-local fix |
44
+ | parser (new) | EPUB parser (spine-order XHTML extraction, delegates to HtmlParser), exported as `deepdoc.EpubParser` | [#13650](https://github.com/infiniflow/ragflow/pull/13650) |
45
+
46
+ Already present before this sync (no action needed): `ast.literal_eval`
47
+ security fix (#12236), XLS/calamine loading (#10660), chartsheet guards
48
+ (#10819), KMeans multi-column detection (#11415 + #12534 cleanup),
49
+ `is_english` regex fix (#11432), OCR GPU memory limit env (#10407),
50
+ `vision_figure_parser_docx_wrapper` guard (#12104).
51
+
52
+ **Deliberately NOT ported** (decisions):
53
+
54
+ - **Go + HTTP microservice pipeline** (#16323, `deepdoc/server/`, `DEEPDOC_URL`
55
+ DLA client): conflicts with this library's pure-Python, in-process design.
56
+ - **Automatic table orientation detection/rotation** (#12719 feature +
57
+ #12981 / #17016 follow-ups): feature not in our baseline; runs 4× OCR per
58
+ table; adopt separately if ever needed.
59
+ - **`rag.utils.lazy_image` memory refactors** (docx/excel lazy image loading,
60
+ #13329 / #13558): optimization only; deferred until vendoring lazy_image is
61
+ justified.
62
+ - **3-tuple outlines** (`extract_pdf_outlines` returning
63
+ `(title, depth, page)`, #10456 TOC feature): our `outlines` stays
64
+ `(title, depth)` to avoid breaking downstream consumers.
65
+ - **`model_speciess` → `model_species` rename** (#13929): our attribute name
66
+ is part of the public surface; renaming would break existing callers.
67
+ - **paddleocr / opendataloader / somark parsers**: cloud-API clients, not
68
+ currently needed; revisit on demand.
69
+ - **MinerU feature evolution** (API-only backends, VLM descriptions, etc.):
70
+ only the security fix was taken; feature sync deferred until MinerU usage
71
+ is confirmed.
72
+ - **Pure refactors** (moving functions between ragflow modules, lefthook
73
+ reformatting): not applicable to our layout.
74
+
75
+ Regression tests: `tests/test_upstream_sync_regressions.py`.
76
+
77
+ ## How to run the next sync
78
+
79
+ ```bash
80
+ # in a ragflow clone
81
+ git log --reverse --format="%h %ad %s" --date=short 18ea0fb7..origin/main -- deepdoc
82
+ ```
83
+
84
+ Then for each commit: understand the behavior change, find the corresponding
85
+ code in this repo, and re-implement it in this library's style (config
86
+ injection, relative imports, vendored `common/` + `depend/`). Update this file.
@@ -27,6 +27,7 @@ __all__ = [
27
27
  "PlainParser",
28
28
  "DocxParser",
29
29
  "DoclingParser",
30
+ "EpubParser",
30
31
  "ExcelParser",
31
32
  "PptParser",
32
33
  "HtmlParser",
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
18
18
  commit_id: str | None
19
19
  __commit_id__: str | None
20
20
 
21
- __version__ = version = '0.2.2'
22
- __version_tuple__ = version_tuple = (0, 2, 2)
21
+ __version__ = version = '0.2.5'
22
+ __version_tuple__ = version_tuple = (0, 2, 5)
23
23
 
24
- __commit_id__ = commit_id = 'g2c122018a'
24
+ __commit_id__ = commit_id = 'g8ef70a65b'
@@ -11,7 +11,7 @@ import requests
11
11
  from openai import OpenAI
12
12
  from openai.lib.azure import AzureOpenAI
13
13
  from PIL import Image
14
- from zhipuai import ZhipuAI
14
+ from zai import ZhipuAiClient
15
15
 
16
16
  # 可选导入 ollama
17
17
  try:
@@ -162,7 +162,12 @@ class Zhipu4V(Base):
162
162
 
163
163
  def __init__(self, key, model_name="glm-4v", lang="Chinese", **kwargs):
164
164
  super().__init__(key, model_name, lang)
165
- self.client = ZhipuAI(api_key=key)
165
+ # Preserve the legacy override and China default instead of allowing
166
+ # the maintained SDK's ZAI_BASE_URL to redirect this provider adapter.
167
+ self.client = ZhipuAiClient(
168
+ api_key=key,
169
+ base_url=kwargs.get("base_url") or os.getenv("ZHIPUAI_BASE_URL") or "https://open.bigmodel.cn/api/paas/v4/",
170
+ )
166
171
 
167
172
  def describe_with_prompt(self, image, prompt=None):
168
173
  """使用智谱AI视觉模型描述图片"""
@@ -301,9 +306,16 @@ class GeminiCV(Base):
301
306
 
302
307
  def __init__(self, key, model_name="gemini-pro-vision", lang="Chinese", **kwargs):
303
308
  super().__init__(key, model_name, lang)
304
- import google.generativeai as genai
305
- genai.configure(api_key=key)
306
- self.model = genai.GenerativeModel(model_name)
309
+ # The legacy google-generativeai SDK caps protobuf below 6, so Gemini
310
+ # uses the maintained google-genai SDK as an optional extra instead.
311
+ try:
312
+ from google import genai
313
+ except ImportError as e:
314
+ raise ImportError(
315
+ "Gemini vision requires the google-genai package. "
316
+ "Install it with: pip install 'deepdoc-lib[gemini]'"
317
+ ) from e
318
+ self.client = genai.Client(api_key=key)
307
319
 
308
320
  def describe_with_prompt(self, image, prompt=None):
309
321
  """使用Google Gemini模型描述图片"""
@@ -318,11 +330,19 @@ class GeminiCV(Base):
318
330
 
319
331
  pil_image = Image.open(BytesIO(img_data))
320
332
 
321
- response = self.model.generate_content([
322
- prompt or self.prompt(image),
323
- pil_image
324
- ])
333
+ response = self.client.models.generate_content(
334
+ model=self.model_name,
335
+ contents=[prompt or self.prompt(image), pil_image],
336
+ )
325
337
 
338
+ # google-genai returns None (the legacy SDK raised) when the
339
+ # response has no text, e.g. a safety block; keep the error path.
340
+ # A block can be reported on the prompt or on the candidate.
341
+ if response.text is None:
342
+ candidates = response.candidates or []
343
+ finish_reason = candidates[0].finish_reason if candidates else None
344
+ finish_reason = getattr(finish_reason, "name", finish_reason)
345
+ raise ValueError(f"Gemini returned no text (finish_reason={finish_reason}, prompt_feedback={response.prompt_feedback})")
326
346
  return response.text.strip()
327
347
  except Exception as e:
328
348
  return "**ERROR**: " + str(e)
@@ -466,4 +486,4 @@ def create_vision_model(config=None):
466
486
  return VisionModelFactory.create_model(config)
467
487
 
468
488
  else:
469
- raise ValueError(f"不支持的配置类型: {type(config)}")
489
+ raise ValueError(f"不支持的配置类型: {type(config)}")
@@ -16,6 +16,7 @@
16
16
 
17
17
  from .docx_parser import RAGFlowDocxParser as DocxParser
18
18
  from .docling_parser import DoclingParser
19
+ from .epub_parser import RAGFlowEpubParser as EpubParser
19
20
  from .excel_parser import RAGFlowExcelParser as ExcelParser
20
21
  from .html_parser import RAGFlowHtmlParser as HtmlParser
21
22
  from .json_parser import RAGFlowJsonParser as JsonParser
@@ -31,6 +32,7 @@ __all__ = [
31
32
  "PlainParser",
32
33
  "DocxParser",
33
34
  "DoclingParser",
35
+ "EpubParser",
34
36
  "ExcelParser",
35
37
  "PptParser",
36
38
  "HtmlParser",
@@ -0,0 +1,139 @@
1
+ #
2
+ # Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ import logging
18
+ import warnings
19
+ import zipfile
20
+ from io import BytesIO
21
+ from xml.etree import ElementTree
22
+
23
+ from ..config import TokenizerConfig
24
+ from .html_parser import RAGFlowHtmlParser
25
+
26
+ # OPF XML namespaces
27
+ _OPF_NS = "http://www.idpf.org/2007/opf"
28
+ _CONTAINER_NS = "urn:oasis:names:tc:opendocument:xmlns:container"
29
+
30
+ # Media types that contain readable XHTML content
31
+ _XHTML_MEDIA_TYPES = {"application/xhtml+xml", "text/html", "text/xml"}
32
+
33
+ logger = logging.getLogger(__name__)
34
+
35
+
36
+ class RAGFlowEpubParser:
37
+ """Parse EPUB files by extracting XHTML content in spine (reading) order
38
+ and delegating to RAGFlowHtmlParser for chunking."""
39
+
40
+ def __init__(self, tokenizer_cfg: TokenizerConfig | None = None):
41
+ self.html_parser = RAGFlowHtmlParser(tokenizer_cfg=tokenizer_cfg)
42
+
43
+ def __call__(self, fnm, binary=None, chunk_token_num=512):
44
+ if binary is not None:
45
+ if not binary:
46
+ logger.warning(
47
+ "RAGFlowEpubParser received an empty EPUB binary payload for %r",
48
+ fnm,
49
+ )
50
+ raise ValueError("Empty EPUB binary payload")
51
+ zf = zipfile.ZipFile(BytesIO(binary))
52
+ else:
53
+ zf = zipfile.ZipFile(fnm)
54
+
55
+ try:
56
+ content_items = self._get_spine_items(zf)
57
+ all_sections = []
58
+
59
+ for item_path in content_items:
60
+ try:
61
+ html_bytes = zf.read(item_path)
62
+ except KeyError:
63
+ continue
64
+ if not html_bytes:
65
+ logger.debug("Skipping empty EPUB content item: %s", item_path)
66
+ continue
67
+ with warnings.catch_warnings():
68
+ warnings.filterwarnings("ignore", category=UserWarning)
69
+ sections = self.html_parser(item_path, binary=html_bytes, chunk_token_num=chunk_token_num)
70
+ all_sections.extend(sections)
71
+
72
+ return all_sections
73
+ finally:
74
+ zf.close()
75
+
76
+ @staticmethod
77
+ def _get_spine_items(zf):
78
+ """Return content file paths in spine (reading) order."""
79
+ # 1. Find the OPF file path from META-INF/container.xml
80
+ try:
81
+ container_xml = zf.read("META-INF/container.xml")
82
+ except KeyError:
83
+ return RAGFlowEpubParser._fallback_xhtml_order(zf)
84
+
85
+ try:
86
+ container_root = ElementTree.fromstring(container_xml)
87
+ except ElementTree.ParseError:
88
+ logger.warning("Failed to parse META-INF/container.xml; falling back to XHTML order.")
89
+ return RAGFlowEpubParser._fallback_xhtml_order(zf)
90
+
91
+ rootfile_el = container_root.find(f".//{{{_CONTAINER_NS}}}rootfile")
92
+ if rootfile_el is None:
93
+ return RAGFlowEpubParser._fallback_xhtml_order(zf)
94
+
95
+ opf_path = rootfile_el.get("full-path", "")
96
+ if not opf_path:
97
+ return RAGFlowEpubParser._fallback_xhtml_order(zf)
98
+
99
+ # Base directory of the OPF file (content paths are relative to it)
100
+ opf_dir = opf_path.rsplit("/", 1)[0] + "/" if "/" in opf_path else ""
101
+
102
+ # 2. Parse the OPF file
103
+ try:
104
+ opf_xml = zf.read(opf_path)
105
+ except KeyError:
106
+ return RAGFlowEpubParser._fallback_xhtml_order(zf)
107
+
108
+ try:
109
+ opf_root = ElementTree.fromstring(opf_xml)
110
+ except ElementTree.ParseError:
111
+ logger.warning("Failed to parse OPF file '%s'; falling back to XHTML order.", opf_path)
112
+ return RAGFlowEpubParser._fallback_xhtml_order(zf)
113
+
114
+ # 3. Build id->href+mediatype map from <manifest>
115
+ manifest = {}
116
+ for item in opf_root.findall(f".//{{{_OPF_NS}}}item"):
117
+ item_id = item.get("id", "")
118
+ href = item.get("href", "")
119
+ media_type = item.get("media-type", "")
120
+ if item_id and href:
121
+ manifest[item_id] = (href, media_type)
122
+
123
+ # 4. Walk <spine> to get reading order
124
+ spine_items = []
125
+ for itemref in opf_root.findall(f".//{{{_OPF_NS}}}itemref"):
126
+ idref = itemref.get("idref", "")
127
+ if idref not in manifest:
128
+ continue
129
+ href, media_type = manifest[idref]
130
+ if media_type not in _XHTML_MEDIA_TYPES:
131
+ continue
132
+ spine_items.append(opf_dir + href)
133
+
134
+ return spine_items if spine_items else RAGFlowEpubParser._fallback_xhtml_order(zf)
135
+
136
+ @staticmethod
137
+ def _fallback_xhtml_order(zf):
138
+ """Fallback: return all .xhtml/.html files sorted alphabetically."""
139
+ return sorted(n for n in zf.namelist() if n.lower().endswith((".xhtml", ".html", ".htm")) and not n.startswith("META-INF/"))