everalgo-parser 0.1.0__tar.gz → 0.2.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/.gitignore +4 -0
  2. {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/CHANGELOG.md +20 -11
  3. {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/PKG-INFO +28 -16
  4. {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/README.md +19 -11
  5. everalgo_parser-0.2.1/docs/architecture.md +307 -0
  6. {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/pyproject.toml +9 -5
  7. {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/src/everalgo/parser/__init__.py +9 -7
  8. {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/src/everalgo/parser/_utils.py +3 -6
  9. {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/src/everalgo/parser/audio.py +1 -2
  10. {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/src/everalgo/parser/document.py +9 -8
  11. {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/src/everalgo/parser/image.py +5 -10
  12. {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/src/everalgo/parser/prompts/en/audio.py +1 -5
  13. {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/src/everalgo/parser/prompts/en/document.py +1 -6
  14. {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/src/everalgo/parser/prompts/en/image.py +1 -5
  15. {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/src/everalgo/parser/url.py +5 -5
  16. everalgo_parser-0.2.1/src/everalgo/parser/video.py +31 -0
  17. {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/parser/test_dispatch.py +1 -1
  18. {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/parser/test_document.py +1 -1
  19. {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/parser/test_public_api.py +1 -14
  20. {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/parser/test_utils.py +4 -4
  21. everalgo_parser-0.1.0/src/everalgo/parser/video.py +0 -38
  22. {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/LICENSE +0 -0
  23. {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/src/everalgo/parser/prompts/__init__.py +0 -0
  24. {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/src/everalgo/parser/prompts/en/__init__.py +0 -0
  25. {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/src/everalgo/parser/prompts/zh/__init__.py +0 -0
  26. {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/src/everalgo/parser/prompts/zh/audio.py +0 -0
  27. {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/src/everalgo/parser/prompts/zh/document.py +0 -0
  28. {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/src/everalgo/parser/prompts/zh/image.py +0 -0
  29. {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/src/everalgo/parser/py.typed +0 -0
  30. {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/fixtures/sample.bmp +0 -0
  31. {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/fixtures/sample.docx +0 -0
  32. {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/fixtures/sample.eml +0 -0
  33. {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/fixtures/sample.htm +0 -0
  34. {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/fixtures/sample.html +0 -0
  35. {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/fixtures/sample.jpg +0 -0
  36. {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/fixtures/sample.pdf +0 -0
  37. {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/fixtures/sample.png +0 -0
  38. {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/fixtures/sample.pptx +0 -0
  39. {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/fixtures/sample.svg +0 -0
  40. {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/fixtures/sample.tiff +0 -0
  41. {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/fixtures/sample.txt +0 -0
  42. {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/fixtures/sample.wav +0 -0
  43. {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/fixtures/sample.webp +0 -0
  44. {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/fixtures/sample.xlsx +0 -0
  45. {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/parser/__init__.py +0 -0
  46. {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/parser/test_audio.py +0 -0
  47. {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/parser/test_e2e_all_formats.py +0 -0
  48. {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/parser/test_e2e_pdf_openrouter.py +0 -0
  49. {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/parser/test_image.py +0 -0
  50. {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/parser/test_url.py +0 -0
  51. {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/parser/test_video.py +0 -0
@@ -56,3 +56,7 @@ local/
56
56
 
57
57
  # Claude Code session-local state (scheduled tasks, agent caches, locks)
58
58
  .claude/
59
+ benchmarks/claudecode
60
+
61
+ # Git worktrees (isolated workspaces)
62
+ .worktrees/
@@ -5,17 +5,24 @@ All notable changes to this distribution will be documented in this file.
5
5
  The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
6
6
  and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
7
7
 
8
- Entries are generated by [git-cliff](https://git-cliff.org/) from Conventional
9
- Commit messages on `main`. See the repository root [`README.md` § Cutting a release](../../README.md#cutting-a-release)
10
- for the full workflow.
11
-
12
8
  <!-- git-cliff-unreleased-start -->
13
9
  ## [Unreleased]
10
+ <!-- git-cliff-unreleased-end -->
11
+
12
+ ## [0.2.1] - 2026-06-15
13
+
14
+ ### Changed
15
+
16
+ - **License relicensed from MIT to Apache-2.0** as part of the pre-open-source security audit.
17
+
18
+ ## [0.2.0] - 2026-05-27
19
+
20
+ > First archived changelog. Entries below accumulated since the initial `0.1.0` PyPI release (published manually, without a git tag or per-package changelog), so this `0.2.0` section also consolidates the previously-unarchived `0.1.0` surface.
14
21
 
15
22
  ### Added
16
23
 
17
- - **PDF parser** — single multimodal LLM call, mirrors
18
- `evermemos-multimodal/src/parser/pdf/gemini_parser.py`.
24
+ - **PDF parser** — single multimodal LLM call, mirrors the upstream
25
+ internal multimodal PDF parser reference implementation.
19
26
  - **Image parser** — single LLM call for normal-ratio images; tall
20
27
  screenshots (height/width > 10) are split into overlapping vertical
21
28
  slices, OCR'd per slice, and merged with a second LLM call using
@@ -40,8 +47,8 @@ for the full workflow.
40
47
  unknown) → HTML handler with Open Graph / Twitter Card / `<meta>` tag
41
48
  extraction into `ParsedContent.metadata`. `file://` and other local-
42
49
  filesystem URIs are rejected (AGENTS.md §1). Inner modality recorded in
43
- `metadata["inner_modality"]`. Metadata-extractor schema lifted from
44
- `evermemos-opensource/src/common_utils/url_extractor.py`.
50
+ `metadata["inner_modality"]`. Metadata-extractor schema lifted from the
51
+ upstream internal URL extractor reference implementation.
45
52
  - **MIME-aware top-level dispatch** — `parser.aparse` now falls back to
46
53
  `mime` when `extension` is missing or `UNKNOWN`, so the canonical
47
54
  `RawFile(uri=..., mime="application/pdf")` example actually routes to
@@ -58,8 +65,8 @@ for the full workflow.
58
65
 
59
66
  ### Notes
60
67
 
61
- - `video` submodule remains a stub — upstream evermemos-multimodal has no
62
- video implementation to port; backend selection pending an ADR.
68
+ - `video` submodule remains a stub — the upstream internal multimodal parser
69
+ has no video implementation to port; backend selection pending an ADR.
63
70
  - The parser layer deliberately omits retry / fallback / multi-key
64
71
  rotation / Prometheus metrics. Those are deployment concerns
65
72
  (ADR-012); algorithm-layer operators surface failures via `LLMError`
@@ -73,4 +80,6 @@ for the full workflow.
73
80
  - Optional `cairosvg >= 2.7.0` via `[svg]` extra.
74
81
  - System: LibreOffice (`soffice` on PATH) for Office document parsing.
75
82
 
76
- <!-- git-cliff-unreleased-end -->
83
+ [Unreleased]: https://github.com/EverMind-AI/EverAlgo/compare/everalgo-parser/v0.2.1...HEAD
84
+ [0.2.1]: https://github.com/EverMind-AI/EverAlgo/compare/everalgo-parser/v0.2.0...everalgo-parser/v0.2.1
85
+ [0.2.0]: https://github.com/EverMind-AI/EverAlgo/releases/tag/everalgo-parser/v0.2.0
@@ -1,22 +1,26 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: everalgo-parser
3
- Version: 0.1.0
3
+ Version: 0.2.1
4
4
  Summary: EverAlgo parser: multimodal raw-file parsing (image / audio / document / video / url) into ParsedContent.
5
5
  Project-URL: Homepage, https://github.com/EverMind-AI/EverAlgo
6
6
  Project-URL: Repository, https://github.com/EverMind-AI/EverAlgo
7
7
  Project-URL: Issues, https://github.com/EverMind-AI/EverAlgo/issues
8
8
  Project-URL: Documentation, https://github.com/EverMind-AI/EverAlgo/tree/main/packages/everalgo-parser
9
9
  Project-URL: Changelog, https://github.com/EverMind-AI/EverAlgo/blob/main/packages/everalgo-parser/CHANGELOG.md
10
- Author: EverMind
11
- License-Expression: MIT
10
+ Author-email: EverMind <Evermind@shanda.com>
11
+ License-Expression: Apache-2.0
12
12
  License-File: LICENSE
13
+ Keywords: everalgo,llm,multimodal,parsing
13
14
  Classifier: Development Status :: 3 - Alpha
14
- Classifier: License :: OSI Approved :: MIT License
15
+ Classifier: Intended Audience :: Developers
16
+ Classifier: License :: OSI Approved :: Apache Software License
17
+ Classifier: Programming Language :: Python :: 3
15
18
  Classifier: Programming Language :: Python :: 3.12
19
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
16
20
  Requires-Python: >=3.12
17
21
  Requires-Dist: asgiref>=3.8.0
18
22
  Requires-Dist: beautifulsoup4>=4.12.0
19
- Requires-Dist: everalgo-core<2.0.0,>=0.1.0
23
+ Requires-Dist: everalgo-core<2.0.0,>=0.2.0
20
24
  Requires-Dist: httpx>=0.27.0
21
25
  Requires-Dist: pillow>=10.0.0
22
26
  Provides-Extra: svg
@@ -25,33 +29,32 @@ Description-Content-Type: text/markdown
25
29
 
26
30
  # everalgo-parser
27
31
 
28
- Multimodal parsing — image / audio / document / video / url raw inputs into `ParsedContent`. Used by `everalgo-knowledge` for file ingestion and by evermem step 1 for inline parsing.
32
+ Multimodal parsing — image / audio / document / video / url raw inputs into `ParsedContent`. Used by `everalgo-knowledge` for file ingestion and by EverOS step 1 for inline parsing.
29
33
 
30
34
  See the umbrella project: [EverAlgo monorepo](../../README.md) and the architecture document at [`docs/concepts/architecture.md`](../../docs/concepts/architecture.md).
31
35
 
32
36
  ## Quick start
33
37
 
34
38
  ```python
35
- import everalgo
36
39
  from everalgo.llm.config import LLMConfig
37
40
  from everalgo.llm.providers.openai_compat import OpenAICompatClient
38
41
  from everalgo.parser import aparse, RawFile
39
42
 
40
- # Configure an LLM once (process-wide). The parser uses OpenAI-compatible
41
- # clients; OpenRouter is the reference deployment (Gemini multimodal via
42
- # OpenRouter passthrough).
43
- everalgo.configure(OpenAICompatClient(LLMConfig(
43
+ # Create an LLM client per call (or share one instance). The parser uses
44
+ # OpenAI-compatible clients; OpenRouter is the reference deployment
45
+ # (Gemini multimodal via OpenRouter passthrough).
46
+ client = OpenAICompatClient(LLMConfig(
44
47
  model="google/gemini-3-flash-preview",
45
48
  api_key="sk-or-v1-...",
46
49
  base_url="https://openrouter.ai/api/v1",
47
- )))
50
+ ))
48
51
 
49
52
  # Bytes-in: caller already hydrated the file.
50
- parsed = await aparse(RawFile(content=pdf_bytes, extension="pdf"))
53
+ parsed = await aparse(RawFile(content=pdf_bytes, extension="pdf"), llm=client)
51
54
  print(parsed.text)
52
55
 
53
56
  # URL-in: parser fetches over HTTP, then delegates to the HTML handler.
54
- parsed = await aparse(RawFile(uri="https://example.com/article"))
57
+ parsed = await aparse(RawFile(uri="https://example.com/article"), llm=client)
55
58
  print(parsed.metadata["title"], parsed.text[:500])
56
59
  ```
57
60
 
@@ -97,8 +100,17 @@ The parser detects `soffice` via `shutil.which("soffice")` and the canonical mac
97
100
  - The library is **stateless**: it never reads the filesystem and never owns business state. HTTP I/O (LLM calls, URL fetching) is explicitly allowed.
98
101
  - No retry / fallback / metrics inside operators — surface failures via `LLMError`, let the caller wrap.
99
102
 
103
+ ## Security notes for callers
104
+
105
+ `everalgo-parser` is a stateless library; production safety properties belong to the calling service. The points below are easy to miss and worth wiring into your integration:
106
+
107
+ - **SSRF — `url.aparse` fetches arbitrary HTTP(S) URIs.** `fetch_uri` rejects non-http(s) schemes (no `file://`, no `gopher://`) but does **not** block private / link-local / loopback IPs (e.g. `127.0.0.1`, `10.0.0.0/8`, `169.254.169.254` cloud metadata, IPv6 ULA). If callers may pass attacker-controlled URLs, resolve the hostname first and reject private ranges before invoking `aparse`, or front the parser with an egress proxy that does the filtering.
108
+ - **Decompression-bomb cap is set at import time.** `everalgo/parser/__init__.py` pins `PIL.Image.MAX_IMAGE_PIXELS = 100_000_000` (10000×10000) — Pillow will raise `DecompressionBombError` above that. Re-assign after import if your workload genuinely needs larger images.
109
+ - **Office conversion runs LibreOffice as a subprocess.** `document._convert_to_pdf_via_soffice` validates the `extension` parameter (alphanumeric, ≤8 chars) before writing the temp file, but the LibreOffice attack surface itself is large. Run the host with a non-root user and treat untrusted office docs as you would any other complex binary input.
110
+ - **`cairosvg` (optional `[svg]` extra) is LGPL-3.0-or-later.** EverAlgo itself is Apache-2.0. LGPLv3 is compatible with Apache-2.0 when `cairosvg` is consumed as an unmodified library through its public Python API (which is what the SVG path does); if you statically vendor or modify `cairosvg`, LGPLv3 §4 / §5 obligations apply to that derivative. Not installing the `[svg]` extra removes the dependency entirely.
111
+
100
112
  ## Reference
101
113
 
102
114
  - Architecture (definitive): [`docs/concepts/architecture.md`](../../docs/concepts/architecture.md)
103
- - Schema source for PDF / image / audio / document / html / email: `evermemos-multimodal` (tag `prod-20260306-0331-v1`).
104
- - Schema source for URL metadata extraction: `evermemos-opensource/src/common_utils/url_extractor.py`.
115
+ - Schema source for PDF / image / audio / document / html / email: upstream internal multimodal parser library (snapshot `prod-20260306-0331-v1`).
116
+ - Schema source for URL metadata extraction: upstream internal URL extractor reference implementation.
@@ -1,32 +1,31 @@
1
1
  # everalgo-parser
2
2
 
3
- Multimodal parsing — image / audio / document / video / url raw inputs into `ParsedContent`. Used by `everalgo-knowledge` for file ingestion and by evermem step 1 for inline parsing.
3
+ Multimodal parsing — image / audio / document / video / url raw inputs into `ParsedContent`. Used by `everalgo-knowledge` for file ingestion and by EverOS step 1 for inline parsing.
4
4
 
5
5
  See the umbrella project: [EverAlgo monorepo](../../README.md) and the architecture document at [`docs/concepts/architecture.md`](../../docs/concepts/architecture.md).
6
6
 
7
7
  ## Quick start
8
8
 
9
9
  ```python
10
- import everalgo
11
10
  from everalgo.llm.config import LLMConfig
12
11
  from everalgo.llm.providers.openai_compat import OpenAICompatClient
13
12
  from everalgo.parser import aparse, RawFile
14
13
 
15
- # Configure an LLM once (process-wide). The parser uses OpenAI-compatible
16
- # clients; OpenRouter is the reference deployment (Gemini multimodal via
17
- # OpenRouter passthrough).
18
- everalgo.configure(OpenAICompatClient(LLMConfig(
14
+ # Create an LLM client per call (or share one instance). The parser uses
15
+ # OpenAI-compatible clients; OpenRouter is the reference deployment
16
+ # (Gemini multimodal via OpenRouter passthrough).
17
+ client = OpenAICompatClient(LLMConfig(
19
18
  model="google/gemini-3-flash-preview",
20
19
  api_key="sk-or-v1-...",
21
20
  base_url="https://openrouter.ai/api/v1",
22
- )))
21
+ ))
23
22
 
24
23
  # Bytes-in: caller already hydrated the file.
25
- parsed = await aparse(RawFile(content=pdf_bytes, extension="pdf"))
24
+ parsed = await aparse(RawFile(content=pdf_bytes, extension="pdf"), llm=client)
26
25
  print(parsed.text)
27
26
 
28
27
  # URL-in: parser fetches over HTTP, then delegates to the HTML handler.
29
- parsed = await aparse(RawFile(uri="https://example.com/article"))
28
+ parsed = await aparse(RawFile(uri="https://example.com/article"), llm=client)
30
29
  print(parsed.metadata["title"], parsed.text[:500])
31
30
  ```
32
31
 
@@ -72,8 +71,17 @@ The parser detects `soffice` via `shutil.which("soffice")` and the canonical mac
72
71
  - The library is **stateless**: it never reads the filesystem and never owns business state. HTTP I/O (LLM calls, URL fetching) is explicitly allowed.
73
72
  - No retry / fallback / metrics inside operators — surface failures via `LLMError`, let the caller wrap.
74
73
 
74
+ ## Security notes for callers
75
+
76
+ `everalgo-parser` is a stateless library; production safety properties belong to the calling service. The points below are easy to miss and worth wiring into your integration:
77
+
78
+ - **SSRF — `url.aparse` fetches arbitrary HTTP(S) URIs.** `fetch_uri` rejects non-http(s) schemes (no `file://`, no `gopher://`) but does **not** block private / link-local / loopback IPs (e.g. `127.0.0.1`, `10.0.0.0/8`, `169.254.169.254` cloud metadata, IPv6 ULA). If callers may pass attacker-controlled URLs, resolve the hostname first and reject private ranges before invoking `aparse`, or front the parser with an egress proxy that does the filtering.
79
+ - **Decompression-bomb cap is set at import time.** `everalgo/parser/__init__.py` pins `PIL.Image.MAX_IMAGE_PIXELS = 100_000_000` (10000×10000) — Pillow will raise `DecompressionBombError` above that. Re-assign after import if your workload genuinely needs larger images.
80
+ - **Office conversion runs LibreOffice as a subprocess.** `document._convert_to_pdf_via_soffice` validates the `extension` parameter (alphanumeric, ≤8 chars) before writing the temp file, but the LibreOffice attack surface itself is large. Run the host with a non-root user and treat untrusted office docs as you would any other complex binary input.
81
+ - **`cairosvg` (optional `[svg]` extra) is LGPL-3.0-or-later.** EverAlgo itself is Apache-2.0. LGPLv3 is compatible with Apache-2.0 when `cairosvg` is consumed as an unmodified library through its public Python API (which is what the SVG path does); if you statically vendor or modify `cairosvg`, LGPLv3 §4 / §5 obligations apply to that derivative. Not installing the `[svg]` extra removes the dependency entirely.
82
+
75
83
  ## Reference
76
84
 
77
85
  - Architecture (definitive): [`docs/concepts/architecture.md`](../../docs/concepts/architecture.md)
78
- - Schema source for PDF / image / audio / document / html / email: `evermemos-multimodal` (tag `prod-20260306-0331-v1`).
79
- - Schema source for URL metadata extraction: `evermemos-opensource/src/common_utils/url_extractor.py`.
86
+ - Schema source for PDF / image / audio / document / html / email: upstream internal multimodal parser library (snapshot `prod-20260306-0331-v1`).
87
+ - Schema source for URL metadata extraction: upstream internal URL extractor reference implementation.
@@ -0,0 +1,307 @@
1
+ # everalgo-parser — 模块 & 接口图
2
+
3
+ 供 code review 用,展示当前实现的模块边界、调用关系、对外接口、类型契约、依赖拓扑。
4
+
5
+ GitHub 默认渲染下面的 Mermaid 图。
6
+
7
+ ---
8
+
9
+ ## 1. 包内文件结构
10
+
11
+ ```
12
+ packages/everalgo-parser/
13
+ ├── src/everalgo/parser/
14
+ │ ├── __init__.py ← 顶层 facade:dispatch + re-export 公开类型
15
+ │ ├── _utils.py ← 私有助手:fetch_uri / decode_bytes / aspect-ratio / html-clean
16
+ │ ├── image.py ← IMAGE modality 算子(含长图分块 + BMP/TIFF 转码 + SVG 栅格化)
17
+ │ ├── audio.py ← AUDIO modality 算子
18
+ │ ├── document.py ← PDF / HTML / EMAIL / Office(LibreOffice)四合一
19
+ │ ├── url.py ← URL modality 算子(HTTP 抓取 + OG 抽取 + 委托 HTML)
20
+ │ ├── video.py ← stub,等 ADR
21
+ │ └── prompts/
22
+ │ ├── __init__.py
23
+ │ ├── en/{image,audio,document}.py ← PROMPT_FOR_{PICTURE,MERGE,AUDIO,FILE,HTML}
24
+ │ └── zh/{image,audio,document}.py ← 中文对偶
25
+ ├── tests/
26
+ │ ├── fixtures/ ← 17 个真实 sample.* 文件
27
+ │ └── parser/
28
+ │ ├── test_audio.py
29
+ │ ├── test_dispatch.py
30
+ │ ├── test_document.py
31
+ │ ├── test_e2e_all_formats.py
32
+ │ ├── test_e2e_pdf_openrouter.py
33
+ │ ├── test_image.py
34
+ │ ├── test_public_api.py
35
+ │ ├── test_url.py
36
+ │ ├── test_utils.py
37
+ │ └── test_video.py
38
+ ├── pyproject.toml
39
+ ├── README.md
40
+ └── CHANGELOG.md
41
+ ```
42
+
43
+ ---
44
+
45
+ ## 2. 公开 API 表面(facade)
46
+
47
+ `from everalgo.parser import ...` 一处导入,2 个函数 + 3 个类型。
48
+
49
+ | 符号 | 种类 | 签名 / 说明 |
50
+ |---|---|---|
51
+ | `aparse` | async fn | `async def aparse(raw_file: RawFile, *, llm: LLMClient \| None = None) -> ParsedContent` |
52
+ | `parse` | sync fn | sync 桥接(`asgiref.async_to_sync`),不要在事件循环里调 |
53
+ | `RawFile`, `ParsedContent`, `Modality` | type re-export | 来自 `everalgo.types` |
54
+
55
+ 5 个 modality 子模块(`image` / `audio` / `document` / `url` / `video`)是顶层 dispatcher 的内部实现,不在 `__all__` 中;通过 Python attribute lookup 仍可访问,但不是公开承诺(对标 sklearn / numpy 模式)。
56
+
57
+ ---
58
+
59
+ ## 3. 顶层 dispatch 流(`parser.aparse`)
60
+
61
+ ```mermaid
62
+ flowchart TD
63
+ A[aparse raw_file] --> B{has content?}
64
+ B -- no --> C{uri is http/https?}
65
+ C -- yes --> URL[url.aparse: fetch then dispatch by Content-Type]
66
+ C -- no --> E1[ValueError: no content, no fetchable uri]
67
+ B -- yes --> M1[try get_modality extension]
68
+ M1 --> M2{UNKNOWN and mime set?}
69
+ M2 -- yes --> M3[fall back to get_modality_from_mime]
70
+ M2 -- no --> M4[use extension-derived modality]
71
+ M3 --> N{Modality}
72
+ M4 --> N
73
+ N -- still UNKNOWN --> E2[ValueError: cannot map to Modality]
74
+ N -- IMAGE --> IMG[image.aparse]
75
+ N -- AUDIO --> AUD[audio.aparse]
76
+ N -- PDF / DOCUMENT / HTML / EMAIL --> DOC[document.aparse]
77
+ N -- DIRECT --> DIR[passthrough: bytes utf-8 decode]
78
+ URL --> R[ParsedContent modality=URL]
79
+ IMG --> R2[ParsedContent modality=IMAGE]
80
+ AUD --> R3[ParsedContent modality=AUDIO]
81
+ DOC --> R4[ParsedContent modality=PDF/DOCUMENT/HTML/EMAIL]
82
+ DIR --> R5[ParsedContent modality=DIRECT]
83
+ ```
84
+
85
+ **关键规则**:
86
+ - `uri=http(s)://` + `content=b""` → 走 `url.aparse`(内部抓取 + 按 Content-Type 再分派)。
87
+ - 已经有 `content`:**extension 优先**,extension 解析不到时**用 mime 兜底**(`get_modality_from_mime`)。这样 design.md §2.1 的 `RawFile(uri=..., mime="application/pdf")` 走 URL 抓取后能再走到 PDF handler。
88
+ - 子模块内部派发用 extension(`_MIME_MAP[ext]` 查 mime)。当顶层用 mime 解出 modality 时,自动 `model_copy(extension=get_extension_from_mime(mime))` 给子模块。
89
+ - `file://` 在 `_utils.fetch_uri` 里直接拒绝(AGENTS.md §1)。
90
+ - `Modality.URL` 是顶层 dispatch 产生的"来源标签",不出现在 extension 派发表里。
91
+
92
+ ---
93
+
94
+ ## 4. 模块依赖图(import 关系)
95
+
96
+ ```mermaid
97
+ graph LR
98
+ PARSER[parser.__init__<br/>facade] --> IMAGE[image.py]
99
+ PARSER --> AUDIO[audio.py]
100
+ PARSER --> DOC[document.py]
101
+ PARSER --> URL[url.py]
102
+ PARSER --> VIDEO[video.py stub]
103
+
104
+ IMAGE --> UTILS[_utils.py]
105
+ IMAGE --> PROMPTS_EN[prompts.en]
106
+ DOC --> UTILS
107
+ DOC --> PROMPTS_EN
108
+ AUDIO --> PROMPTS_EN
109
+ URL --> UTILS
110
+ URL -. lazy .-> DOC
111
+
112
+ IMAGE --> CORE[everalgo-core:<br/>llm + types]
113
+ AUDIO --> CORE
114
+ DOC --> CORE
115
+ URL --> CORE
116
+
117
+ IMAGE -.-> PIL[Pillow]
118
+ IMAGE -.-> CAIRO[cairosvg<br/>optional svg extra]
119
+ DOC -.-> BS4[beautifulsoup4]
120
+ DOC -.-> SOFFICE[soffice<br/>system dep]
121
+ UTILS -.-> BS4
122
+ UTILS -.-> PIL
123
+ UTILS -.-> HTTPX[httpx]
124
+ ```
125
+
126
+ **说明**:
127
+ - `url.py` → `document.py` 是 **lazy import**(写在函数内,避免循环 import)
128
+ - 5 个 parser 模块都依赖 `everalgo-core`(LLM 调用 + 类型)
129
+ - `cairosvg` 是 `[svg]` extra,延迟 import(只在解析 svg 时触发)
130
+ - `soffice` 是**系统包**,运行时 detection(`shutil.which`)
131
+
132
+ ---
133
+
134
+ ## 5. 类型契约
135
+
136
+ ### `RawFile`(输入)
137
+
138
+ | 字段 | 类型 | 默认 | 说明 |
139
+ |---|---|---|---|
140
+ | `content` | `bytes` | `b""` | 文件 bytes。可选,但 `content` / `uri` 至少有一个 |
141
+ | `mime` | `str` | `""` | MIME hint,如 `application/pdf` |
142
+ | `extension` | `str` | `""` | 不带点小写扩展名,如 `pdf`,**派发依据** |
143
+ | `uri` | `str` | `""` | `http`/`https` 才会被抓取;`file://` 拒绝 |
144
+
145
+ ### `ParsedContent`(输出)
146
+
147
+ | 字段 | 类型 | 说明 |
148
+ |---|---|---|
149
+ | `text` | `str` | 提取后的主文本(Markdown / 纯文本) |
150
+ | `modality` | `Modality` | 来源分类 |
151
+ | `mime` | `str` | 源 MIME(无则空串) |
152
+ | `metadata` | `dict[str, Any]` | 按模态填:`model` / `finish_reason` / `tall_image_parts` / `inline_image_ocr` / `og_tags` / `intermediate_pdf_bytes` 等 |
153
+
154
+ ### `Modality` 枚举(StrEnum,9 值)
155
+
156
+ ```
157
+ IMAGE PDF AUDIO DOCUMENT HTML EMAIL URL DIRECT UNKNOWN
158
+ ```
159
+
160
+ `get_modality(ext: str) -> Modality` 派发函数 + `EXTENSION_TO_MODALITY` 映射,都在 `everalgo-core/types/modality.py`。
161
+
162
+ ---
163
+
164
+ ## 6. 各 modality 调用链
165
+
166
+ ### 6.1 IMAGE(`image.aparse`)
167
+
168
+ ```mermaid
169
+ flowchart LR
170
+ A[bytes + ext] --> B{ext in<br/>bmp/tif/tiff?}
171
+ B -- yes --> C[PIL transcode → PNG]
172
+ B -- no --> D{ext == svg?}
173
+ D -- yes --> E[cairosvg<br/>rasterise → PNG]
174
+ D -- no --> F[as-is]
175
+ C --> G[check_aspect_ratio]
176
+ E --> G
177
+ F --> G
178
+ G --> H{ratio > 10?}
179
+ H -- no --> I[single LLM call<br/>PROMPT_FOR_PICTURE]
180
+ H -- yes --> J[split_image_with_overlap<br/>N parts]
181
+ J --> K[per-part OCR<br/>N x LLM calls]
182
+ K --> L{N > 1?}
183
+ L -- no --> M[return part]
184
+ L -- yes --> N[LLM merge<br/>PROMPT_FOR_MERGE]
185
+ I --> R[ParsedContent]
186
+ M --> R
187
+ N --> R
188
+ ```
189
+
190
+ ### 6.2 DOCUMENT(`document.aparse`)
191
+
192
+ ```mermaid
193
+ flowchart TD
194
+ A[bytes + ext] --> M{Modality from ext}
195
+ M -- PDF --> P[image_url part + PROMPT_FOR_FILE → 1 LLM call]
196
+ M -- HTML --> H[decode_bytes → clean_html_for_llm bs4 → text part + PROMPT_FOR_HTML → 1 LLM call]
197
+ M -- EMAIL --> E[stdlib email parse → headers + body]
198
+ E --> EI{inline images?}
199
+ EI -- yes --> EOCR[per-image OCR via LLM<br/>PROMPT_FOR_PICTURE]
200
+ EOCR --> ES[substitute cid placeholders]
201
+ EI -- no --> ES2[text only]
202
+ M -- DOCUMENT Office --> O[soffice convert-to pdf]
203
+ O --> P2[reuse PDF path]
204
+ ```
205
+
206
+ ### 6.3 AUDIO(`audio.aparse`)
207
+
208
+ ```
209
+ RawFile.content (mp3/wav/m4a/...) → image_url data URI + PROMPT_FOR_AUDIO → 1 LLM call → ParsedContent.text
210
+ ```
211
+
212
+ ### 6.4 URL(`url.aparse`)
213
+
214
+ ```mermaid
215
+ flowchart LR
216
+ A[uri or content] --> B{has content?}
217
+ B -- no --> C[fetch_uri httpx]
218
+ C -- http/https --> D[bytes + Content-Type]
219
+ C -- other scheme --> X[ValueError reject]
220
+ B -- yes --> D
221
+ D --> CT[get_modality_from_mime fetched Content-Type]
222
+ CT --> SW{inner Modality}
223
+ SW -- IMAGE --> IM[image.aparse]
224
+ SW -- AUDIO --> AU[audio.aparse]
225
+ SW -- PDF / DOCUMENT / EMAIL --> DC[document.aparse]
226
+ SW -- HTML / UNKNOWN-fallback --> HT[document.aparse extension=html]
227
+ HT -.OG/meta extract.-> META[og_tags / twitter_tags / favicon / ...]
228
+ IM --> R[ParsedContent modality=URL]
229
+ AU --> R
230
+ DC --> R
231
+ HT --> R
232
+ META --> R
233
+ ```
234
+
235
+ **关键规则**:
236
+ - 抓回来的 Content-Type **驱动 inner dispatch**。`application/pdf` → PDF handler;`image/png` → image handler;`audio/mpeg` → audio handler;`text/html` 或未知 → HTML handler(兜底)。
237
+ - **OG / Twitter / `<meta>` 抽取仅 HTML 响应做** —— 非 HTML 响应的 `ParsedContent.metadata` 不带 `og_tags` / `title` 等字段。
238
+ - 返回的 `modality` 永远是 `Modality.URL`,真实的 inner 类型放在 `metadata["inner_modality"]`,调用方既能知道"这是从网络来的",也能知道"实际是 PDF"。
239
+
240
+ ---
241
+
242
+ ## 7. 依赖拓扑
243
+
244
+ ### Pip 依赖(`pyproject.toml`)
245
+
246
+ | 依赖 | 用途 | 何时拉 |
247
+ |---|---|---|
248
+ | `everalgo-core` | LLM 客户端 + 公共类型 + fake_llm | 必装 |
249
+ | `asgiref` | sync 桥接(`async_to_sync`)| 必装 |
250
+ | `pillow` | image 转码 + 长图分块 | 必装 |
251
+ | `beautifulsoup4` | HTML 清洗 + URL OG 抽取 | 必装 |
252
+ | `httpx` | URL HTTP 抓取 | 透传(`everalgo-core` 已经依赖 openai SDK 拉了) |
253
+ | `cairosvg` | SVG 栅格化 | `pip install 'everalgo-parser[svg]'` 才装 |
254
+
255
+ ### 系统依赖
256
+
257
+ | 程序 | 用途 | 检测 |
258
+ |---|---|---|
259
+ | `soffice`(LibreOffice)| Office 文档转 PDF | `shutil.which("soffice")` + macOS `/Applications/LibreOffice.app/Contents/MacOS/soffice` |
260
+
261
+ ---
262
+
263
+ ## 8. 测试覆盖
264
+
265
+ | 文件 | 数量 | 类型 | LLM |
266
+ |---|---|---|---|
267
+ | `test_public_api.py` | 2 | 公开 API 形状 | 不用 |
268
+ | `test_dispatch.py` | 25 | 顶层派发 | `FakeLLMClient` |
269
+ | `test_image.py` | 12 | PIL 路径 + 长图分块 / 合并 | `FakeLLMClient` |
270
+ | `test_audio.py` | 5 | audio 子模块 happy + error path | `FakeLLMClient` |
271
+ | `test_document.py` | 12 | PDF / HTML / EML 单测 | `FakeLLMClient` |
272
+ | `test_url.py` | 21 | OG 抽取 + httpx mock | `FakeLLMClient` + `respx` |
273
+ | `test_utils.py` | 18 | `_utils` 助手(no LLM,no network)| 不用 |
274
+ | `test_video.py` | 2 | video stub NotImplementedError | 不用 |
275
+ | `tests/types/test_modality.py` | 12 | Modality enum + get_modality | 不用 |
276
+ | `tests/types/test_parsed.py` | 3 | ParsedContent 字段 | 不用 |
277
+ | `test_e2e_all_formats.py` | 7 | 全格式 e2e | 真 LLM(env `OPENROUTER_API_KEY`)|
278
+ | `test_e2e_pdf_openrouter.py` | 1 | PDF spike e2e | 真 LLM(env `OPENROUTER_API_KEY`)|
279
+
280
+ **测试总数**:`391`(parser+core)/`923`(workspace 全量)
281
+ **E2E**(需 `OPENROUTER_API_KEY`):`8`(`test_e2e_all_formats.py` 7 + `test_e2e_pdf_openrouter.py` 1)
282
+
283
+ ---
284
+
285
+ ## 9. 公开规范契约
286
+
287
+ | 规范 | 对应 |
288
+ |---|---|
289
+ | `a` 前缀代表 async | ✅ `aparse` async / `parse` sync 桥接(ADR-010) |
290
+ | Protocol over ABC | ✅ 没有 `ABC` 基类,LLMClient 是 Protocol(ADR-011) |
291
+ | 无 retry / fallback / metrics | ✅ 算法层不做工程高可用(ADR-012) |
292
+ | 无状态(不读文件系统) | ✅ 接 `bytes` / HTTP,`file://` 拒绝(AGENTS.md §1) |
293
+ | 懒 `%`-format 日志 | ✅ `logger.debug("count=%d", n)` 风格(ADR-013) |
294
+ | Prompt = `prompts/{en,zh}/<name>.py` 模块常量 | ✅ |
295
+ | Numpy-style docstring | ✅ |
296
+ | 英文 docstring / 标识符 / commit | ✅(docs/ 下 design.md 例外,中文 OK) |
297
+
298
+ ---
299
+
300
+ ## 10. 已知未实现 / 推迟
301
+
302
+ | 项 | 状态 | 备注 |
303
+ |---|---|---|
304
+ | `video.py` | stub | 上游内部多模态解析库**也没有**实现,Gemini Video vs Whisper+帧抽取 选型待 ADR |
305
+ | `audio` BMP-like 转码 | 没做 | mp3/wav 测过;m4a/amr/aiff/aac/ogg/flac 未在 e2e 覆盖,Gemini 兼容度未全测 |
306
+ | `Office` 系统依赖动态检测降级 | 仅运行时报错 | 没有 import 时友好警告;`_find_soffice` 缺失就 `RuntimeError` |
307
+ | `T1` 主条目其他 schema | 待 RESOLVED | 我们只 RESOLVED 了 `T1.parser` 子集(`ParsedContent` / `Modality` / `RawFile`) |
@@ -4,19 +4,23 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "everalgo-parser"
7
- version = "0.1.0"
7
+ version = "0.2.1"
8
8
  description = "EverAlgo parser: multimodal raw-file parsing (image / audio / document / video / url) into ParsedContent."
9
9
  requires-python = ">=3.12"
10
- license = "MIT"
10
+ license = "Apache-2.0"
11
11
  readme = "README.md"
12
- authors = [{ name = "EverMind" }]
12
+ authors = [{ name = "EverMind", email = "Evermind@shanda.com" }]
13
+ keywords = ["llm", "multimodal", "parsing", "everalgo"]
13
14
  classifiers = [
14
15
  "Development Status :: 3 - Alpha",
15
- "License :: OSI Approved :: MIT License",
16
+ "Intended Audience :: Developers",
17
+ "License :: OSI Approved :: Apache Software License",
18
+ "Programming Language :: Python :: 3",
16
19
  "Programming Language :: Python :: 3.12",
20
+ "Topic :: Scientific/Engineering :: Artificial Intelligence",
17
21
  ]
18
22
  dependencies = [
19
- "everalgo-core>=0.1.0,<2.0.0",
23
+ "everalgo-core>=0.2.0,<2.0.0",
20
24
  "asgiref>=3.8.0",
21
25
  "pillow>=10.0.0",
22
26
  "beautifulsoup4>=4.12.0",
@@ -3,7 +3,6 @@
3
3
  Public surface (re-exports for ergonomic ``from everalgo.parser import ...``):
4
4
 
5
5
  - Functions: ``aparse`` / ``parse`` — dispatch entry points.
6
- - Submodules: ``image`` / ``audio`` / ``document`` / ``url`` / ``video``.
7
6
  - Types: ``ParsedContent`` / ``Modality`` / ``RawFile`` (re-exported from
8
7
  ``everalgo.types`` for one-stop import).
9
8
  """
@@ -15,8 +14,9 @@ from typing import TYPE_CHECKING
15
14
  from urllib.parse import urlparse
16
15
 
17
16
  from asgiref.sync import async_to_sync
17
+ from PIL import Image
18
18
 
19
- from everalgo.parser import audio, document, image, url, video
19
+ from everalgo.parser import audio, document, image, url
20
20
  from everalgo.types import (
21
21
  Modality,
22
22
  ParsedContent,
@@ -34,17 +34,19 @@ __all__ = [
34
34
  "ParsedContent",
35
35
  "RawFile",
36
36
  "aparse",
37
- "audio",
38
- "document",
39
- "image",
40
37
  "parse",
41
- "url",
42
- "video",
43
38
  ]
44
39
 
45
40
  logger = logging.getLogger(__name__)
46
41
  logger.addHandler(logging.NullHandler())
47
42
 
43
+ # Explicit decompression-bomb cap (Pillow default raises ``DecompressionBombError``
44
+ # only at ~2x this threshold). 100M pixels = 10000x10000, ample for any realistic
45
+ # screenshot / scan while keeping a hostile caller from exhausting memory via a
46
+ # crafted PNG/TIFF. Applications can override by re-assigning
47
+ # ``PIL.Image.MAX_IMAGE_PIXELS`` after importing the parser.
48
+ Image.MAX_IMAGE_PIXELS = 100_000_000
49
+
48
50
 
49
51
  async def aparse(raw_file: RawFile, *, llm: LLMClient) -> ParsedContent:
50
52
  """Top-level dispatch: classify the input and route to a submodule.
@@ -1,16 +1,13 @@
1
1
  """Internal parser helpers.
2
2
 
3
- Migrated (with English docstrings) from
4
- ``evermemos-multimodal/src/parser/utils.py`` (prod-20260306-0331-v1):
5
-
6
3
  - ``decode_bytes`` — multi-encoding decoder for unknown-charset bytes.
7
4
  - ``check_aspect_ratio`` / ``split_image_with_overlap`` — long-image (tall
8
5
  screenshot) handling for the image parser.
9
6
  - ``clean_html_content`` — strict text extraction (drops all tags).
10
7
  - ``clean_html_for_llm`` — preserves structural tags inside ``<body>``,
11
8
  strips known-noise tags and useless attributes, returns HTML.
12
- - ``fetch_uri`` — HTTP(S) fetch for the URL parser. Adapted from
13
- ``evermemos-opensource/src/common_utils/url_extractor.py``.
9
+ - ``fetch_uri`` — HTTP(S) fetch for the URL parser. Adapted from an
10
+ upstream internal URL extractor reference implementation.
14
11
  """
15
12
 
16
13
  from __future__ import annotations
@@ -43,7 +40,7 @@ _DEFAULT_USER_AGENT = "Mozilla/5.0 (compatible; everalgo-parser/0.1; +https://ev
43
40
  async def fetch_uri(uri: str, *, timeout: float = 30.0) -> tuple[bytes, str]:
44
41
  """Fetch ``http``/``https`` URI and return ``(content_bytes, content_type)``.
45
42
 
46
- Adapted from ``evermemos-opensource/src/common_utils/url_extractor.py``;
43
+ Adapted from an upstream internal URL extractor reference implementation;
47
44
  rejects non-http(s) schemes per AGENTS.md §1 (no filesystem access).
48
45
 
49
46
  Parameters
@@ -1,7 +1,6 @@
1
1
  """Audio parser — ASR via multimodal LLM.
2
2
 
3
- Mirrors ``evermemos-multimodal/src/parser/audio/gemini_parser.py``: send the
4
- audio bytes as a single ``image_url`` data URI (OpenRouter / Gemini accepts
3
+ Sends the audio bytes as a single ``image_url`` data URI (OpenRouter / Gemini accepts
5
4
  audio MIMEs through the same content-part shape) and the LLM transcribes.
6
5
  """
7
6
 
@@ -4,8 +4,7 @@ Per ``AGENTS.md`` and the engineering alignment for the parser migration, the
4
4
  ``document`` submodule covers every non-image / non-audio file shape:
5
5
 
6
6
  - ``Modality.PDF`` — sent as a single ``application/pdf`` data URI to the
7
- multimodal LLM (mirrors the path used in
8
- ``evermemos-multimodal/src/parser/pdf/gemini_parser.py``).
7
+ multimodal LLM.
9
8
  - ``Modality.HTML`` — bs4-cleaned (``_utils.clean_html_for_llm``) before
10
9
  sending the structured HTML to the LLM with the HTML prompt.
11
10
  - ``Modality.EMAIL`` — pure-python ``email`` stdlib parsing; when inline
@@ -50,8 +49,7 @@ _PDF_MIME = "application/pdf"
50
49
  _HTML_MAX_INPUT_CHARS = 1_000_000
51
50
  _MACOS_SOFFICE = "/Applications/LibreOffice.app/Contents/MacOS/soffice"
52
51
 
53
- # Verbatim from ``evermemos-multimodal/src/parser/email/eml_parser.py``:
54
- # inline image OCR caps at 8K tokens (single-image scope, no need for the
52
+ # Inline image OCR caps at 8K tokens (single-image scope, no need for the
55
53
  # 32K merge budget used by the long-image stitcher).
56
54
  _INLINE_OCR_MAX_TOKENS = 8_000
57
55
  _INLINE_IMAGE_MIMETYPES = frozenset(
@@ -160,8 +158,6 @@ async def _aparse_html(raw_file: RawFile, *, llm: LLMClient) -> ParsedContent:
160
158
  async def _aparse_email(raw_file: RawFile, *, llm: LLMClient) -> ParsedContent:
161
159
  """EML → ``ParsedContent`` via stdlib ``email`` + inline-image OCR.
162
160
 
163
- Mirrors evermemos-multimodal/src/parser/email/eml_parser.py:
164
-
165
161
  1. Extract headers.
166
162
  2. Collect inline images (Content-ID → bytes).
167
163
  3. Extract body: prefer ``text/plain`` when there are no cid images
@@ -261,7 +257,7 @@ async def _aparse_office(
261
257
 
262
258
 
263
259
  def _extract_email_headers(msg: EmailMessage) -> str:
264
- lines = []
260
+ lines: list[str] = []
265
261
  for field in ("Subject", "From", "To", "Cc", "Date"):
266
262
  value = msg.get(field)
267
263
  if value:
@@ -367,7 +363,7 @@ async def _ocr_inline_images(
367
363
  except Exception as exc:
368
364
  # Best-effort per-image: a single OCR failure should not abort the
369
365
  # whole email parse — log and continue with a placeholder marker.
370
- logger.warning("OCR failed for inline image %s: %s", filename, exc)
366
+ logger.warning("OCR failed for inline image %s: %s", filename, exc, exc_info=True)
371
367
  results[idx] = f"[image: {filename}, OCR failed]"
372
368
  return results
373
369
 
@@ -403,6 +399,11 @@ async def _convert_to_pdf_via_soffice(
403
399
  content: bytes, extension: str, soffice: str
404
400
  ) -> bytes: # pragma: no cover (libreoffice binary, not in CI)
405
401
  """Run LibreOffice headless to convert bytes → PDF bytes."""
402
+ # Reject path-traversal / directory separators in `extension` so a hostile
403
+ # caller cannot escape the tempdir via e.g. ``extension="../../etc/passwd"``.
404
+ # `extension` is expected to be a short ASCII suffix (docx / xlsx / ...).
405
+ if not extension or not extension.isalnum() or len(extension) > 8:
406
+ raise ValueError(f"invalid extension for soffice conversion: {extension!r}")
406
407
  with tempfile.TemporaryDirectory() as tmpdir: # pragma: no cover
407
408
  in_path = Path(tmpdir) / f"input.{extension}" # pragma: no cover
408
409
  in_path.write_bytes(content) # pragma: no cover
@@ -1,7 +1,5 @@
1
1
  """Image parser — multimodal LLM OCR with tall-image splitting.
2
2
 
3
- Mirrors ``evermemos-multimodal/src/parser/image/gemini_parser.py``:
4
-
5
3
  - Normal-ratio images go through a single LLM call.
6
4
  - Tall screenshots (height/width > 10) are split into overlapping slices
7
5
  via ``_utils.split_image_with_overlap``; each slice is OCR'd independently
@@ -54,10 +52,8 @@ _SVG_EXTENSIONS: frozenset[str] = frozenset({"svg"})
54
52
 
55
53
  _TALL_IMAGE_RATIO_THRESHOLD = 10.0
56
54
 
57
- # Token budgets — verbatim from
58
- # ``evermemos-multimodal/src/parser/image/gemini_parser.py``. Per-slice OCR
59
- # caps at 8K; the merge step gets 4x because it joins multiple OCR'd chunks
60
- # and the merged text is longer than any individual slice.
55
+ # Token budgets. Per-slice OCR caps at 8K; the merge step gets 4x because it
56
+ # joins multiple OCR'd chunks and the merged text is longer than any individual slice.
61
57
  _MAX_OCR_OUTPUT_TOKENS = 8_000
62
58
  _MAX_MERGE_OUTPUT_TOKENS = 32_000
63
59
 
@@ -138,9 +134,6 @@ async def _aparse_tall(
138
134
  ) -> ParsedContent:
139
135
  r"""Split-then-merge OCR for tall screenshots.
140
136
 
141
- Mirrors ``evermemos-multimodal/src/parser/image/base.py:parse`` +
142
- ``gemini_parser._merge_parts``:
143
-
144
137
  1. Split into N overlapping vertical slices.
145
138
  2. OCR each slice; **drop empty results** so the merge step sees only
146
139
  real text (an all-empty slice from a blank PDF region would otherwise
@@ -223,7 +216,9 @@ async def _aparse_tall(
223
216
  merge_fallback_used = True
224
217
  last_response = merge_resp
225
218
  except Exception as exc:
226
- logger.warning("LLM merge of tall-image OCR parts failed (%s); falling back to deterministic join", exc)
219
+ logger.warning(
220
+ "LLM merge of tall-image OCR parts failed (%s); falling back to deterministic join", exc, exc_info=True
221
+ )
227
222
  merged_text = "\n\n".join(ocr_results)
228
223
  merge_fallback_used = True
229
224
 
@@ -1,8 +1,4 @@
1
- """Audio parser prompt.
2
-
3
- Lifted verbatim from ``evermemos-multimodal/src/parser/prompts.py``
4
- (prod-20260306-0331-v1).
5
- """
1
+ """Audio parser prompt."""
6
2
 
7
3
  PROMPT_FOR_AUDIO = """This is a voice message sent by a user in a chat application. Transcribe it word by word.
8
4
 
@@ -1,9 +1,4 @@
1
- """Document parser prompts (PDF / Office / HTML).
2
-
3
- Lifted verbatim from ``evermemos-multimodal/src/parser/prompts.py``
4
- (prod-20260306-0331-v1) where these strings have been iterated on against
5
- real customer documents.
6
- """
1
+ """Document parser prompts (PDF / Office / HTML)."""
7
2
 
8
3
  PROMPT_FOR_FILE = """Read this document and return its content as if you were reading it naturally.
9
4
 
@@ -1,8 +1,4 @@
1
- """Image parser prompts.
2
-
3
- Lifted verbatim from ``evermemos-multimodal/src/parser/prompts.py``
4
- (prod-20260306-0331-v1).
5
- """
1
+ """Image parser prompts."""
6
2
 
7
3
  PROMPT_FOR_PICTURE = """Read this image and return its content as if you were reading it naturally.
8
4
 
@@ -18,10 +18,10 @@ The returned ``ParsedContent.modality`` is always ``Modality.URL`` so
18
18
  callers can distinguish "loaded from network" from "passed in as bytes";
19
19
  the inner modality is recorded in ``metadata["inner_modality"]``.
20
20
 
21
- Schema source for the metadata extractor:
22
- ``evermemos-opensource/src/common_utils/url_extractor.py``
23
- (``_extract_og_tags`` / ``_extract_twitter_tags`` / ``_extract_meta_tags`` /
24
- ``_extract_title`` / ``_extract_favicon``).
21
+ Schema source for the metadata extractor: an upstream internal URL
22
+ extractor reference implementation (``_extract_og_tags`` /
23
+ ``_extract_twitter_tags`` / ``_extract_meta_tags`` / ``_extract_title`` /
24
+ ``_extract_favicon``).
25
25
  """
26
26
 
27
27
  from __future__ import annotations
@@ -166,7 +166,7 @@ async def _dispatch_inner(
166
166
  raise ValueError(f"url.aparse: cannot dispatch fetched body for modality={modality.value!r}")
167
167
 
168
168
 
169
- # ---- metadata extraction (port from evermemos-opensource) ----
169
+ # ---- metadata extraction (port from upstream internal URL extractor) ----
170
170
 
171
171
 
172
172
  def _extract_metadata(html_text: str, *, base_url: str) -> dict[str, object]:
@@ -0,0 +1,31 @@
1
+ """Video parser — deferred.
2
+
3
+ No implementation yet; selection between Gemini Video and Whisper + frame
4
+ extraction is a follow-up decision. Calling ``aparse`` / ``parse`` today
5
+ raises ``NotImplementedError`` so dispatch in ``everalgo.parser.aparse``
6
+ surfaces the deferred status cleanly to callers.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from typing import TYPE_CHECKING
12
+
13
+ if TYPE_CHECKING:
14
+ from everalgo.llm import LLMClient
15
+ from everalgo.types import ParsedContent, RawFile
16
+
17
+ __all__ = ["aparse", "parse"]
18
+
19
+
20
+ async def aparse(raw_file: RawFile, *, llm: LLMClient) -> ParsedContent:
21
+ """Parse video → ``ParsedContent``. Deferred — see module docstring."""
22
+ raise NotImplementedError(
23
+ "video parser is deferred; selection between Gemini Video / Whisper + frame extraction pending an ADR."
24
+ )
25
+
26
+
27
+ def parse(raw_file: RawFile, *, llm: LLMClient) -> ParsedContent:
28
+ """Sync bridge. Deferred — see module docstring."""
29
+ raise NotImplementedError(
30
+ "video parser is deferred; selection between Gemini Video / Whisper + frame extraction pending an ADR."
31
+ )
@@ -246,7 +246,7 @@ async def test_aparse_raises_for_unsupported_modality(monkeypatch: pytest.Monkey
246
246
  from everalgo.types import Modality
247
247
 
248
248
  # Modality.URL is valid but not handled by the dispatch switch (IMAGE/AUDIO/PDF/etc.)
249
- monkeypatch.setattr(pkg, "get_modality", lambda _ext: Modality.URL)
249
+ monkeypatch.setattr(pkg, "get_modality", lambda _ext: Modality.URL) # type: ignore[misc]
250
250
  with pytest.raises(ValueError, match="Unsupported modality"):
251
251
  await parser_pkg.aparse(RawFile(content=b"x", extension="mp4"), llm=_fake())
252
252
 
@@ -130,7 +130,7 @@ async def test_aparse_html_empty_after_clean_returns_empty_parsed_content(monkey
130
130
  import everalgo.parser.document as doc_mod
131
131
 
132
132
  # Monkeypatch clean_html_for_llm to return "" so the empty-after-cleanup branch fires
133
- monkeypatch.setattr(doc_mod, "clean_html_for_llm", lambda _: "")
133
+ monkeypatch.setattr(doc_mod, "clean_html_for_llm", lambda _: "") # type: ignore[misc]
134
134
 
135
135
  fake = _make_fake()
136
136
  html = b"<html><body><p>some content</p></body></html>"
@@ -10,16 +10,8 @@ def test_top_level_routing_aparse_parse() -> None:
10
10
  assert callable(parse)
11
11
 
12
12
 
13
- def test_five_submodules_have_dual_interface() -> None:
14
- from everalgo.parser import audio, document, image, url, video
15
-
16
- for mod in (image, audio, document, video, url):
17
- assert inspect.iscoroutinefunction(mod.aparse)
18
- assert callable(mod.parse)
19
-
20
-
21
13
  def test_dunder_all_lists_public_symbols() -> None:
22
- """Re-exports cover the 5 submodules + dispatch fns + 3 facade types."""
14
+ """``__all__`` lists only facade-level symbols; submodules are internal."""
23
15
  from everalgo.parser import __all__
24
16
 
25
17
  assert sorted(__all__) == sorted(
@@ -28,11 +20,6 @@ def test_dunder_all_lists_public_symbols() -> None:
28
20
  "ParsedContent",
29
21
  "RawFile",
30
22
  "aparse",
31
- "audio",
32
- "document",
33
- "image",
34
23
  "parse",
35
- "url",
36
- "video",
37
24
  ]
38
25
  )
@@ -27,26 +27,26 @@ def _png_bytes(width: int, height: int) -> bytes:
27
27
  def test_check_aspect_ratio_normal_image_not_tall() -> None:
28
28
  needs_split, ratio = check_aspect_ratio(_png_bytes(100, 200))
29
29
  assert needs_split is False
30
- assert ratio == pytest.approx(2.0)
30
+ assert ratio == pytest.approx(2.0) # pyright: ignore[reportUnknownMemberType]
31
31
 
32
32
 
33
33
  def test_check_aspect_ratio_tall_image_needs_split() -> None:
34
34
  needs_split, ratio = check_aspect_ratio(_png_bytes(100, 2000))
35
35
  assert needs_split is True
36
- assert ratio == pytest.approx(20.0)
36
+ assert ratio == pytest.approx(20.0) # pyright: ignore[reportUnknownMemberType]
37
37
 
38
38
 
39
39
  def test_check_aspect_ratio_custom_max_ratio() -> None:
40
40
  # 200 / 100 = 2.0; with max_ratio=1.5 it should need split
41
41
  needs_split, ratio = check_aspect_ratio(_png_bytes(100, 200), max_ratio=1.5)
42
42
  assert needs_split is True
43
- assert ratio == pytest.approx(2.0)
43
+ assert ratio == pytest.approx(2.0) # pyright: ignore[reportUnknownMemberType]
44
44
 
45
45
 
46
46
  def test_check_aspect_ratio_square_image() -> None:
47
47
  needs_split, ratio = check_aspect_ratio(_png_bytes(100, 100))
48
48
  assert needs_split is False
49
- assert ratio == pytest.approx(1.0)
49
+ assert ratio == pytest.approx(1.0) # pyright: ignore[reportUnknownMemberType]
50
50
 
51
51
 
52
52
  # ---- split_image_with_overlap ----
@@ -1,38 +0,0 @@
1
- """Video parser — deferred.
2
-
3
- evermemos-multimodal has no video parser implementation; the upstream
4
- ``FILE_TYPE_DEFAULT_EXTENSION`` map reserves ``"video": "mp4"`` but no
5
- parser ever consumed it. Selection between Gemini Video and Whisper +
6
- frame extraction is a follow-up decision; when that lands, this stub is
7
- replaced with the real handler. Calling ``aparse`` / ``parse`` today raises
8
- ``NotImplementedError`` so dispatch in ``everalgo.parser.aparse`` surfaces
9
- the deferred status cleanly to callers.
10
- """
11
-
12
- from __future__ import annotations
13
-
14
- from typing import TYPE_CHECKING
15
-
16
- if TYPE_CHECKING:
17
- from everalgo.llm import LLMClient
18
- from everalgo.types import ParsedContent, RawFile
19
-
20
- __all__ = ["aparse", "parse"]
21
-
22
-
23
- async def aparse(raw_file: RawFile, *, llm: LLMClient) -> ParsedContent:
24
- """Parse video → ``ParsedContent``. Deferred — see module docstring."""
25
- raise NotImplementedError(
26
- "video parser is deferred; selection between Gemini Video / Whisper "
27
- "+ frame extraction pending an ADR. No upstream implementation exists "
28
- "in evermemos-multimodal to port."
29
- )
30
-
31
-
32
- def parse(raw_file: RawFile, *, llm: LLMClient) -> ParsedContent:
33
- """Sync bridge. Deferred — see module docstring."""
34
- raise NotImplementedError(
35
- "video parser is deferred; selection between Gemini Video / Whisper "
36
- "+ frame extraction pending an ADR. No upstream implementation exists "
37
- "in evermemos-multimodal to port."
38
- )
File without changes