everalgo-parser 0.1.0__tar.gz → 0.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/.gitignore +4 -0
- {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/CHANGELOG.md +20 -11
- {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/PKG-INFO +28 -16
- {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/README.md +19 -11
- everalgo_parser-0.2.1/docs/architecture.md +307 -0
- {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/pyproject.toml +9 -5
- {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/src/everalgo/parser/__init__.py +9 -7
- {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/src/everalgo/parser/_utils.py +3 -6
- {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/src/everalgo/parser/audio.py +1 -2
- {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/src/everalgo/parser/document.py +9 -8
- {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/src/everalgo/parser/image.py +5 -10
- {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/src/everalgo/parser/prompts/en/audio.py +1 -5
- {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/src/everalgo/parser/prompts/en/document.py +1 -6
- {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/src/everalgo/parser/prompts/en/image.py +1 -5
- {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/src/everalgo/parser/url.py +5 -5
- everalgo_parser-0.2.1/src/everalgo/parser/video.py +31 -0
- {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/parser/test_dispatch.py +1 -1
- {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/parser/test_document.py +1 -1
- {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/parser/test_public_api.py +1 -14
- {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/parser/test_utils.py +4 -4
- everalgo_parser-0.1.0/src/everalgo/parser/video.py +0 -38
- {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/LICENSE +0 -0
- {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/src/everalgo/parser/prompts/__init__.py +0 -0
- {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/src/everalgo/parser/prompts/en/__init__.py +0 -0
- {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/src/everalgo/parser/prompts/zh/__init__.py +0 -0
- {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/src/everalgo/parser/prompts/zh/audio.py +0 -0
- {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/src/everalgo/parser/prompts/zh/document.py +0 -0
- {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/src/everalgo/parser/prompts/zh/image.py +0 -0
- {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/src/everalgo/parser/py.typed +0 -0
- {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/fixtures/sample.bmp +0 -0
- {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/fixtures/sample.docx +0 -0
- {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/fixtures/sample.eml +0 -0
- {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/fixtures/sample.htm +0 -0
- {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/fixtures/sample.html +0 -0
- {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/fixtures/sample.jpg +0 -0
- {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/fixtures/sample.pdf +0 -0
- {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/fixtures/sample.png +0 -0
- {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/fixtures/sample.pptx +0 -0
- {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/fixtures/sample.svg +0 -0
- {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/fixtures/sample.tiff +0 -0
- {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/fixtures/sample.txt +0 -0
- {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/fixtures/sample.wav +0 -0
- {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/fixtures/sample.webp +0 -0
- {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/fixtures/sample.xlsx +0 -0
- {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/parser/__init__.py +0 -0
- {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/parser/test_audio.py +0 -0
- {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/parser/test_e2e_all_formats.py +0 -0
- {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/parser/test_e2e_pdf_openrouter.py +0 -0
- {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/parser/test_image.py +0 -0
- {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/parser/test_url.py +0 -0
- {everalgo_parser-0.1.0 → everalgo_parser-0.2.1}/tests/parser/test_video.py +0 -0
|
@@ -5,17 +5,24 @@ All notable changes to this distribution will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
-
Entries are generated by [git-cliff](https://git-cliff.org/) from Conventional
|
|
9
|
-
Commit messages on `main`. See the repository root [`README.md` § Cutting a release](../../README.md#cutting-a-release)
|
|
10
|
-
for the full workflow.
|
|
11
|
-
|
|
12
8
|
<!-- git-cliff-unreleased-start -->
|
|
13
9
|
## [Unreleased]
|
|
10
|
+
<!-- git-cliff-unreleased-end -->
|
|
11
|
+
|
|
12
|
+
## [0.2.1] - 2026-06-15
|
|
13
|
+
|
|
14
|
+
### Changed
|
|
15
|
+
|
|
16
|
+
- **License relicensed from MIT to Apache-2.0** as part of the pre-open-source security audit.
|
|
17
|
+
|
|
18
|
+
## [0.2.0] - 2026-05-27
|
|
19
|
+
|
|
20
|
+
> First archived changelog. Entries below accumulated since the initial `0.1.0` PyPI release (published manually, without a git tag or per-package changelog), so this `0.2.0` section also consolidates the previously-unarchived `0.1.0` surface.
|
|
14
21
|
|
|
15
22
|
### Added
|
|
16
23
|
|
|
17
|
-
- **PDF parser** — single multimodal LLM call, mirrors
|
|
18
|
-
|
|
24
|
+
- **PDF parser** — single multimodal LLM call, mirrors the upstream
|
|
25
|
+
internal multimodal PDF parser reference implementation.
|
|
19
26
|
- **Image parser** — single LLM call for normal-ratio images; tall
|
|
20
27
|
screenshots (height/width > 10) are split into overlapping vertical
|
|
21
28
|
slices, OCR'd per slice, and merged with a second LLM call using
|
|
@@ -40,8 +47,8 @@ for the full workflow.
|
|
|
40
47
|
unknown) → HTML handler with Open Graph / Twitter Card / `<meta>` tag
|
|
41
48
|
extraction into `ParsedContent.metadata`. `file://` and other local-
|
|
42
49
|
filesystem URIs are rejected (AGENTS.md §1). Inner modality recorded in
|
|
43
|
-
`metadata["inner_modality"]`. Metadata-extractor schema lifted from
|
|
44
|
-
|
|
50
|
+
`metadata["inner_modality"]`. Metadata-extractor schema lifted from the
|
|
51
|
+
upstream internal URL extractor reference implementation.
|
|
45
52
|
- **MIME-aware top-level dispatch** — `parser.aparse` now falls back to
|
|
46
53
|
`mime` when `extension` is missing or `UNKNOWN`, so the canonical
|
|
47
54
|
`RawFile(uri=..., mime="application/pdf")` example actually routes to
|
|
@@ -58,8 +65,8 @@ for the full workflow.
|
|
|
58
65
|
|
|
59
66
|
### Notes
|
|
60
67
|
|
|
61
|
-
- `video` submodule remains a stub — upstream
|
|
62
|
-
video implementation to port; backend selection pending an ADR.
|
|
68
|
+
- `video` submodule remains a stub — the upstream internal multimodal parser
|
|
69
|
+
has no video implementation to port; backend selection pending an ADR.
|
|
63
70
|
- The parser layer deliberately omits retry / fallback / multi-key
|
|
64
71
|
rotation / Prometheus metrics. Those are deployment concerns
|
|
65
72
|
(ADR-012); algorithm-layer operators surface failures via `LLMError`
|
|
@@ -73,4 +80,6 @@ for the full workflow.
|
|
|
73
80
|
- Optional `cairosvg >= 2.7.0` via `[svg]` extra.
|
|
74
81
|
- System: LibreOffice (`soffice` on PATH) for Office document parsing.
|
|
75
82
|
|
|
76
|
-
|
|
83
|
+
[Unreleased]: https://github.com/EverMind-AI/EverAlgo/compare/everalgo-parser/v0.2.1...HEAD
|
|
84
|
+
[0.2.1]: https://github.com/EverMind-AI/EverAlgo/compare/everalgo-parser/v0.2.0...everalgo-parser/v0.2.1
|
|
85
|
+
[0.2.0]: https://github.com/EverMind-AI/EverAlgo/releases/tag/everalgo-parser/v0.2.0
|
|
@@ -1,22 +1,26 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: everalgo-parser
|
|
3
|
-
Version: 0.1
|
|
3
|
+
Version: 0.2.1
|
|
4
4
|
Summary: EverAlgo parser: multimodal raw-file parsing (image / audio / document / video / url) into ParsedContent.
|
|
5
5
|
Project-URL: Homepage, https://github.com/EverMind-AI/EverAlgo
|
|
6
6
|
Project-URL: Repository, https://github.com/EverMind-AI/EverAlgo
|
|
7
7
|
Project-URL: Issues, https://github.com/EverMind-AI/EverAlgo/issues
|
|
8
8
|
Project-URL: Documentation, https://github.com/EverMind-AI/EverAlgo/tree/main/packages/everalgo-parser
|
|
9
9
|
Project-URL: Changelog, https://github.com/EverMind-AI/EverAlgo/blob/main/packages/everalgo-parser/CHANGELOG.md
|
|
10
|
-
Author: EverMind
|
|
11
|
-
License-Expression:
|
|
10
|
+
Author-email: EverMind <Evermind@shanda.com>
|
|
11
|
+
License-Expression: Apache-2.0
|
|
12
12
|
License-File: LICENSE
|
|
13
|
+
Keywords: everalgo,llm,multimodal,parsing
|
|
13
14
|
Classifier: Development Status :: 3 - Alpha
|
|
14
|
-
Classifier:
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
18
|
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
16
20
|
Requires-Python: >=3.12
|
|
17
21
|
Requires-Dist: asgiref>=3.8.0
|
|
18
22
|
Requires-Dist: beautifulsoup4>=4.12.0
|
|
19
|
-
Requires-Dist: everalgo-core<2.0.0,>=0.
|
|
23
|
+
Requires-Dist: everalgo-core<2.0.0,>=0.2.0
|
|
20
24
|
Requires-Dist: httpx>=0.27.0
|
|
21
25
|
Requires-Dist: pillow>=10.0.0
|
|
22
26
|
Provides-Extra: svg
|
|
@@ -25,33 +29,32 @@ Description-Content-Type: text/markdown
|
|
|
25
29
|
|
|
26
30
|
# everalgo-parser
|
|
27
31
|
|
|
28
|
-
Multimodal parsing — image / audio / document / video / url raw inputs into `ParsedContent`. Used by `everalgo-knowledge` for file ingestion and by
|
|
32
|
+
Multimodal parsing — image / audio / document / video / url raw inputs into `ParsedContent`. Used by `everalgo-knowledge` for file ingestion and by EverOS step 1 for inline parsing.
|
|
29
33
|
|
|
30
34
|
See the umbrella project: [EverAlgo monorepo](../../README.md) and the architecture document at [`docs/concepts/architecture.md`](../../docs/concepts/architecture.md).
|
|
31
35
|
|
|
32
36
|
## Quick start
|
|
33
37
|
|
|
34
38
|
```python
|
|
35
|
-
import everalgo
|
|
36
39
|
from everalgo.llm.config import LLMConfig
|
|
37
40
|
from everalgo.llm.providers.openai_compat import OpenAICompatClient
|
|
38
41
|
from everalgo.parser import aparse, RawFile
|
|
39
42
|
|
|
40
|
-
#
|
|
41
|
-
# clients; OpenRouter is the reference deployment
|
|
42
|
-
# OpenRouter passthrough).
|
|
43
|
-
|
|
43
|
+
# Create an LLM client per call (or share one instance). The parser uses
|
|
44
|
+
# OpenAI-compatible clients; OpenRouter is the reference deployment
|
|
45
|
+
# (Gemini multimodal via OpenRouter passthrough).
|
|
46
|
+
client = OpenAICompatClient(LLMConfig(
|
|
44
47
|
model="google/gemini-3-flash-preview",
|
|
45
48
|
api_key="sk-or-v1-...",
|
|
46
49
|
base_url="https://openrouter.ai/api/v1",
|
|
47
|
-
))
|
|
50
|
+
))
|
|
48
51
|
|
|
49
52
|
# Bytes-in: caller already hydrated the file.
|
|
50
|
-
parsed = await aparse(RawFile(content=pdf_bytes, extension="pdf"))
|
|
53
|
+
parsed = await aparse(RawFile(content=pdf_bytes, extension="pdf"), llm=client)
|
|
51
54
|
print(parsed.text)
|
|
52
55
|
|
|
53
56
|
# URL-in: parser fetches over HTTP, then delegates to the HTML handler.
|
|
54
|
-
parsed = await aparse(RawFile(uri="https://example.com/article"))
|
|
57
|
+
parsed = await aparse(RawFile(uri="https://example.com/article"), llm=client)
|
|
55
58
|
print(parsed.metadata["title"], parsed.text[:500])
|
|
56
59
|
```
|
|
57
60
|
|
|
@@ -97,8 +100,17 @@ The parser detects `soffice` via `shutil.which("soffice")` and the canonical mac
|
|
|
97
100
|
- The library is **stateless**: it never reads the filesystem and never owns business state. HTTP I/O (LLM calls, URL fetching) is explicitly allowed.
|
|
98
101
|
- No retry / fallback / metrics inside operators — surface failures via `LLMError`, let the caller wrap.
|
|
99
102
|
|
|
103
|
+
## Security notes for callers
|
|
104
|
+
|
|
105
|
+
`everalgo-parser` is a stateless library; production safety properties belong to the calling service. The points below are easy to miss and worth wiring into your integration:
|
|
106
|
+
|
|
107
|
+
- **SSRF — `url.aparse` fetches arbitrary HTTP(S) URIs.** `fetch_uri` rejects non-http(s) schemes (no `file://`, no `gopher://`) but does **not** block private / link-local / loopback IPs (e.g. `127.0.0.1`, `10.0.0.0/8`, `169.254.169.254` cloud metadata, IPv6 ULA). If callers may pass attacker-controlled URLs, resolve the hostname first and reject private ranges before invoking `aparse`, or front the parser with an egress proxy that does the filtering.
|
|
108
|
+
- **Decompression-bomb cap is set at import time.** `everalgo/parser/__init__.py` pins `PIL.Image.MAX_IMAGE_PIXELS = 100_000_000` (10000×10000) — Pillow will raise `DecompressionBombError` above that. Re-assign after import if your workload genuinely needs larger images.
|
|
109
|
+
- **Office conversion runs LibreOffice as a subprocess.** `document._convert_to_pdf_via_soffice` validates the `extension` parameter (alphanumeric, ≤8 chars) before writing the temp file, but the LibreOffice attack surface itself is large. Run the host with a non-root user and treat untrusted office docs as you would any other complex binary input.
|
|
110
|
+
- **`cairosvg` (optional `[svg]` extra) is LGPL-3.0-or-later.** EverAlgo itself is Apache-2.0. LGPLv3 is compatible with Apache-2.0 when `cairosvg` is consumed as an unmodified library through its public Python API (which is what the SVG path does); if you statically vendor or modify `cairosvg`, LGPLv3 §4 / §5 obligations apply to that derivative. Not installing the `[svg]` extra removes the dependency entirely.
|
|
111
|
+
|
|
100
112
|
## Reference
|
|
101
113
|
|
|
102
114
|
- Architecture (definitive): [`docs/concepts/architecture.md`](../../docs/concepts/architecture.md)
|
|
103
|
-
- Schema source for PDF / image / audio / document / html / email:
|
|
104
|
-
- Schema source for URL metadata extraction:
|
|
115
|
+
- Schema source for PDF / image / audio / document / html / email: upstream internal multimodal parser library (snapshot `prod-20260306-0331-v1`).
|
|
116
|
+
- Schema source for URL metadata extraction: upstream internal URL extractor reference implementation.
|
|
@@ -1,32 +1,31 @@
|
|
|
1
1
|
# everalgo-parser
|
|
2
2
|
|
|
3
|
-
Multimodal parsing — image / audio / document / video / url raw inputs into `ParsedContent`. Used by `everalgo-knowledge` for file ingestion and by
|
|
3
|
+
Multimodal parsing — image / audio / document / video / url raw inputs into `ParsedContent`. Used by `everalgo-knowledge` for file ingestion and by EverOS step 1 for inline parsing.
|
|
4
4
|
|
|
5
5
|
See the umbrella project: [EverAlgo monorepo](../../README.md) and the architecture document at [`docs/concepts/architecture.md`](../../docs/concepts/architecture.md).
|
|
6
6
|
|
|
7
7
|
## Quick start
|
|
8
8
|
|
|
9
9
|
```python
|
|
10
|
-
import everalgo
|
|
11
10
|
from everalgo.llm.config import LLMConfig
|
|
12
11
|
from everalgo.llm.providers.openai_compat import OpenAICompatClient
|
|
13
12
|
from everalgo.parser import aparse, RawFile
|
|
14
13
|
|
|
15
|
-
#
|
|
16
|
-
# clients; OpenRouter is the reference deployment
|
|
17
|
-
# OpenRouter passthrough).
|
|
18
|
-
|
|
14
|
+
# Create an LLM client per call (or share one instance). The parser uses
|
|
15
|
+
# OpenAI-compatible clients; OpenRouter is the reference deployment
|
|
16
|
+
# (Gemini multimodal via OpenRouter passthrough).
|
|
17
|
+
client = OpenAICompatClient(LLMConfig(
|
|
19
18
|
model="google/gemini-3-flash-preview",
|
|
20
19
|
api_key="sk-or-v1-...",
|
|
21
20
|
base_url="https://openrouter.ai/api/v1",
|
|
22
|
-
))
|
|
21
|
+
))
|
|
23
22
|
|
|
24
23
|
# Bytes-in: caller already hydrated the file.
|
|
25
|
-
parsed = await aparse(RawFile(content=pdf_bytes, extension="pdf"))
|
|
24
|
+
parsed = await aparse(RawFile(content=pdf_bytes, extension="pdf"), llm=client)
|
|
26
25
|
print(parsed.text)
|
|
27
26
|
|
|
28
27
|
# URL-in: parser fetches over HTTP, then delegates to the HTML handler.
|
|
29
|
-
parsed = await aparse(RawFile(uri="https://example.com/article"))
|
|
28
|
+
parsed = await aparse(RawFile(uri="https://example.com/article"), llm=client)
|
|
30
29
|
print(parsed.metadata["title"], parsed.text[:500])
|
|
31
30
|
```
|
|
32
31
|
|
|
@@ -72,8 +71,17 @@ The parser detects `soffice` via `shutil.which("soffice")` and the canonical mac
|
|
|
72
71
|
- The library is **stateless**: it never reads the filesystem and never owns business state. HTTP I/O (LLM calls, URL fetching) is explicitly allowed.
|
|
73
72
|
- No retry / fallback / metrics inside operators — surface failures via `LLMError`, let the caller wrap.
|
|
74
73
|
|
|
74
|
+
## Security notes for callers
|
|
75
|
+
|
|
76
|
+
`everalgo-parser` is a stateless library; production safety properties belong to the calling service. The points below are easy to miss and worth wiring into your integration:
|
|
77
|
+
|
|
78
|
+
- **SSRF — `url.aparse` fetches arbitrary HTTP(S) URIs.** `fetch_uri` rejects non-http(s) schemes (no `file://`, no `gopher://`) but does **not** block private / link-local / loopback IPs (e.g. `127.0.0.1`, `10.0.0.0/8`, `169.254.169.254` cloud metadata, IPv6 ULA). If callers may pass attacker-controlled URLs, resolve the hostname first and reject private ranges before invoking `aparse`, or front the parser with an egress proxy that does the filtering.
|
|
79
|
+
- **Decompression-bomb cap is set at import time.** `everalgo/parser/__init__.py` pins `PIL.Image.MAX_IMAGE_PIXELS = 100_000_000` (10000×10000) — Pillow will raise `DecompressionBombError` above that. Re-assign after import if your workload genuinely needs larger images.
|
|
80
|
+
- **Office conversion runs LibreOffice as a subprocess.** `document._convert_to_pdf_via_soffice` validates the `extension` parameter (alphanumeric, ≤8 chars) before writing the temp file, but the LibreOffice attack surface itself is large. Run the host with a non-root user and treat untrusted office docs as you would any other complex binary input.
|
|
81
|
+
- **`cairosvg` (optional `[svg]` extra) is LGPL-3.0-or-later.** EverAlgo itself is Apache-2.0. LGPLv3 is compatible with Apache-2.0 when `cairosvg` is consumed as an unmodified library through its public Python API (which is what the SVG path does); if you statically vendor or modify `cairosvg`, LGPLv3 §4 / §5 obligations apply to that derivative. Not installing the `[svg]` extra removes the dependency entirely.
|
|
82
|
+
|
|
75
83
|
## Reference
|
|
76
84
|
|
|
77
85
|
- Architecture (definitive): [`docs/concepts/architecture.md`](../../docs/concepts/architecture.md)
|
|
78
|
-
- Schema source for PDF / image / audio / document / html / email:
|
|
79
|
-
- Schema source for URL metadata extraction:
|
|
86
|
+
- Schema source for PDF / image / audio / document / html / email: upstream internal multimodal parser library (snapshot `prod-20260306-0331-v1`).
|
|
87
|
+
- Schema source for URL metadata extraction: upstream internal URL extractor reference implementation.
|
|
@@ -0,0 +1,307 @@
|
|
|
1
|
+
# everalgo-parser — 模块 & 接口图
|
|
2
|
+
|
|
3
|
+
供 code review 用,展示当前实现的模块边界、调用关系、对外接口、类型契约、依赖拓扑。
|
|
4
|
+
|
|
5
|
+
GitHub 默认渲染下面的 Mermaid 图。
|
|
6
|
+
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
## 1. 包内文件结构
|
|
10
|
+
|
|
11
|
+
```
|
|
12
|
+
packages/everalgo-parser/
|
|
13
|
+
├── src/everalgo/parser/
|
|
14
|
+
│ ├── __init__.py ← 顶层 facade:dispatch + re-export 公开类型
|
|
15
|
+
│ ├── _utils.py ← 私有助手:fetch_uri / decode_bytes / aspect-ratio / html-clean
|
|
16
|
+
│ ├── image.py ← IMAGE modality 算子(含长图分块 + BMP/TIFF 转码 + SVG 栅格化)
|
|
17
|
+
│ ├── audio.py ← AUDIO modality 算子
|
|
18
|
+
│ ├── document.py ← PDF / HTML / EMAIL / Office(LibreOffice)四合一
|
|
19
|
+
│ ├── url.py ← URL modality 算子(HTTP 抓取 + OG 抽取 + 委托 HTML)
|
|
20
|
+
│ ├── video.py ← stub,等 ADR
|
|
21
|
+
│ └── prompts/
|
|
22
|
+
│ ├── __init__.py
|
|
23
|
+
│ ├── en/{image,audio,document}.py ← PROMPT_FOR_{PICTURE,MERGE,AUDIO,FILE,HTML}
|
|
24
|
+
│ └── zh/{image,audio,document}.py ← 中文对偶
|
|
25
|
+
├── tests/
|
|
26
|
+
│ ├── fixtures/ ← 17 个真实 sample.* 文件
|
|
27
|
+
│ └── parser/
|
|
28
|
+
│ ├── test_audio.py
|
|
29
|
+
│ ├── test_dispatch.py
|
|
30
|
+
│ ├── test_document.py
|
|
31
|
+
│ ├── test_e2e_all_formats.py
|
|
32
|
+
│ ├── test_e2e_pdf_openrouter.py
|
|
33
|
+
│ ├── test_image.py
|
|
34
|
+
│ ├── test_public_api.py
|
|
35
|
+
│ ├── test_url.py
|
|
36
|
+
│ ├── test_utils.py
|
|
37
|
+
│ └── test_video.py
|
|
38
|
+
├── pyproject.toml
|
|
39
|
+
├── README.md
|
|
40
|
+
└── CHANGELOG.md
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
---
|
|
44
|
+
|
|
45
|
+
## 2. 公开 API 表面(facade)
|
|
46
|
+
|
|
47
|
+
`from everalgo.parser import ...` 一处导入,2 个函数 + 3 个类型。
|
|
48
|
+
|
|
49
|
+
| 符号 | 种类 | 签名 / 说明 |
|
|
50
|
+
|---|---|---|
|
|
51
|
+
| `aparse` | async fn | `async def aparse(raw_file: RawFile, *, llm: LLMClient \| None = None) -> ParsedContent` |
|
|
52
|
+
| `parse` | sync fn | sync 桥接(`asgiref.async_to_sync`),不要在事件循环里调 |
|
|
53
|
+
| `RawFile`, `ParsedContent`, `Modality` | type re-export | 来自 `everalgo.types` |
|
|
54
|
+
|
|
55
|
+
5 个 modality 子模块(`image` / `audio` / `document` / `url` / `video`)是顶层 dispatcher 的内部实现,不在 `__all__` 中;通过 Python attribute lookup 仍可访问,但不是公开承诺(对标 sklearn / numpy 模式)。
|
|
56
|
+
|
|
57
|
+
---
|
|
58
|
+
|
|
59
|
+
## 3. 顶层 dispatch 流(`parser.aparse`)
|
|
60
|
+
|
|
61
|
+
```mermaid
|
|
62
|
+
flowchart TD
|
|
63
|
+
A[aparse raw_file] --> B{has content?}
|
|
64
|
+
B -- no --> C{uri is http/https?}
|
|
65
|
+
C -- yes --> URL[url.aparse: fetch then dispatch by Content-Type]
|
|
66
|
+
C -- no --> E1[ValueError: no content, no fetchable uri]
|
|
67
|
+
B -- yes --> M1[try get_modality extension]
|
|
68
|
+
M1 --> M2{UNKNOWN and mime set?}
|
|
69
|
+
M2 -- yes --> M3[fall back to get_modality_from_mime]
|
|
70
|
+
M2 -- no --> M4[use extension-derived modality]
|
|
71
|
+
M3 --> N{Modality}
|
|
72
|
+
M4 --> N
|
|
73
|
+
N -- still UNKNOWN --> E2[ValueError: cannot map to Modality]
|
|
74
|
+
N -- IMAGE --> IMG[image.aparse]
|
|
75
|
+
N -- AUDIO --> AUD[audio.aparse]
|
|
76
|
+
N -- PDF / DOCUMENT / HTML / EMAIL --> DOC[document.aparse]
|
|
77
|
+
N -- DIRECT --> DIR[passthrough: bytes utf-8 decode]
|
|
78
|
+
URL --> R[ParsedContent modality=URL]
|
|
79
|
+
IMG --> R2[ParsedContent modality=IMAGE]
|
|
80
|
+
AUD --> R3[ParsedContent modality=AUDIO]
|
|
81
|
+
DOC --> R4[ParsedContent modality=PDF/DOCUMENT/HTML/EMAIL]
|
|
82
|
+
DIR --> R5[ParsedContent modality=DIRECT]
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
**关键规则**:
|
|
86
|
+
- `uri=http(s)://` + `content=b""` → 走 `url.aparse`(内部抓取 + 按 Content-Type 再分派)。
|
|
87
|
+
- 已经有 `content`:**extension 优先**,extension 解析不到时**用 mime 兜底**(`get_modality_from_mime`)。这样 design.md §2.1 的 `RawFile(uri=..., mime="application/pdf")` 走 URL 抓取后能再走到 PDF handler。
|
|
88
|
+
- 子模块内部派发用 extension(`_MIME_MAP[ext]` 查 mime)。当顶层用 mime 解出 modality 时,自动 `model_copy(extension=get_extension_from_mime(mime))` 给子模块。
|
|
89
|
+
- `file://` 在 `_utils.fetch_uri` 里直接拒绝(AGENTS.md §1)。
|
|
90
|
+
- `Modality.URL` 是顶层 dispatch 产生的"来源标签",不出现在 extension 派发表里。
|
|
91
|
+
|
|
92
|
+
---
|
|
93
|
+
|
|
94
|
+
## 4. 模块依赖图(import 关系)
|
|
95
|
+
|
|
96
|
+
```mermaid
|
|
97
|
+
graph LR
|
|
98
|
+
PARSER[parser.__init__<br/>facade] --> IMAGE[image.py]
|
|
99
|
+
PARSER --> AUDIO[audio.py]
|
|
100
|
+
PARSER --> DOC[document.py]
|
|
101
|
+
PARSER --> URL[url.py]
|
|
102
|
+
PARSER --> VIDEO[video.py stub]
|
|
103
|
+
|
|
104
|
+
IMAGE --> UTILS[_utils.py]
|
|
105
|
+
IMAGE --> PROMPTS_EN[prompts.en]
|
|
106
|
+
DOC --> UTILS
|
|
107
|
+
DOC --> PROMPTS_EN
|
|
108
|
+
AUDIO --> PROMPTS_EN
|
|
109
|
+
URL --> UTILS
|
|
110
|
+
URL -. lazy .-> DOC
|
|
111
|
+
|
|
112
|
+
IMAGE --> CORE[everalgo-core:<br/>llm + types]
|
|
113
|
+
AUDIO --> CORE
|
|
114
|
+
DOC --> CORE
|
|
115
|
+
URL --> CORE
|
|
116
|
+
|
|
117
|
+
IMAGE -.-> PIL[Pillow]
|
|
118
|
+
IMAGE -.-> CAIRO[cairosvg<br/>optional svg extra]
|
|
119
|
+
DOC -.-> BS4[beautifulsoup4]
|
|
120
|
+
DOC -.-> SOFFICE[soffice<br/>system dep]
|
|
121
|
+
UTILS -.-> BS4
|
|
122
|
+
UTILS -.-> PIL
|
|
123
|
+
UTILS -.-> HTTPX[httpx]
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
**说明**:
|
|
127
|
+
- `url.py` → `document.py` 是 **lazy import**(写在函数内,避免循环 import)
|
|
128
|
+
- 5 个 parser 模块都依赖 `everalgo-core`(LLM 调用 + 类型)
|
|
129
|
+
- `cairosvg` 是 `[svg]` extra,延迟 import(只在解析 svg 时触发)
|
|
130
|
+
- `soffice` 是**系统包**,运行时 detection(`shutil.which`)
|
|
131
|
+
|
|
132
|
+
---
|
|
133
|
+
|
|
134
|
+
## 5. 类型契约
|
|
135
|
+
|
|
136
|
+
### `RawFile`(输入)
|
|
137
|
+
|
|
138
|
+
| 字段 | 类型 | 默认 | 说明 |
|
|
139
|
+
|---|---|---|---|
|
|
140
|
+
| `content` | `bytes` | `b""` | 文件 bytes。可选,但 `content` / `uri` 至少有一个 |
|
|
141
|
+
| `mime` | `str` | `""` | MIME hint,如 `application/pdf` |
|
|
142
|
+
| `extension` | `str` | `""` | 不带点小写扩展名,如 `pdf`,**派发依据** |
|
|
143
|
+
| `uri` | `str` | `""` | `http`/`https` 才会被抓取;`file://` 拒绝 |
|
|
144
|
+
|
|
145
|
+
### `ParsedContent`(输出)
|
|
146
|
+
|
|
147
|
+
| 字段 | 类型 | 说明 |
|
|
148
|
+
|---|---|---|
|
|
149
|
+
| `text` | `str` | 提取后的主文本(Markdown / 纯文本) |
|
|
150
|
+
| `modality` | `Modality` | 来源分类 |
|
|
151
|
+
| `mime` | `str` | 源 MIME(无则空串) |
|
|
152
|
+
| `metadata` | `dict[str, Any]` | 按模态填:`model` / `finish_reason` / `tall_image_parts` / `inline_image_ocr` / `og_tags` / `intermediate_pdf_bytes` 等 |
|
|
153
|
+
|
|
154
|
+
### `Modality` 枚举(StrEnum,9 值)
|
|
155
|
+
|
|
156
|
+
```
|
|
157
|
+
IMAGE PDF AUDIO DOCUMENT HTML EMAIL URL DIRECT UNKNOWN
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
`get_modality(ext: str) -> Modality` 派发函数 + `EXTENSION_TO_MODALITY` 映射,都在 `everalgo-core/types/modality.py`。
|
|
161
|
+
|
|
162
|
+
---
|
|
163
|
+
|
|
164
|
+
## 6. 各 modality 调用链
|
|
165
|
+
|
|
166
|
+
### 6.1 IMAGE(`image.aparse`)
|
|
167
|
+
|
|
168
|
+
```mermaid
|
|
169
|
+
flowchart LR
|
|
170
|
+
A[bytes + ext] --> B{ext in<br/>bmp/tif/tiff?}
|
|
171
|
+
B -- yes --> C[PIL transcode → PNG]
|
|
172
|
+
B -- no --> D{ext == svg?}
|
|
173
|
+
D -- yes --> E[cairosvg<br/>rasterise → PNG]
|
|
174
|
+
D -- no --> F[as-is]
|
|
175
|
+
C --> G[check_aspect_ratio]
|
|
176
|
+
E --> G
|
|
177
|
+
F --> G
|
|
178
|
+
G --> H{ratio > 10?}
|
|
179
|
+
H -- no --> I[single LLM call<br/>PROMPT_FOR_PICTURE]
|
|
180
|
+
H -- yes --> J[split_image_with_overlap<br/>N parts]
|
|
181
|
+
J --> K[per-part OCR<br/>N x LLM calls]
|
|
182
|
+
K --> L{N > 1?}
|
|
183
|
+
L -- no --> M[return part]
|
|
184
|
+
L -- yes --> N[LLM merge<br/>PROMPT_FOR_MERGE]
|
|
185
|
+
I --> R[ParsedContent]
|
|
186
|
+
M --> R
|
|
187
|
+
N --> R
|
|
188
|
+
```
|
|
189
|
+
|
|
190
|
+
### 6.2 DOCUMENT(`document.aparse`)
|
|
191
|
+
|
|
192
|
+
```mermaid
|
|
193
|
+
flowchart TD
|
|
194
|
+
A[bytes + ext] --> M{Modality from ext}
|
|
195
|
+
M -- PDF --> P[image_url part + PROMPT_FOR_FILE → 1 LLM call]
|
|
196
|
+
M -- HTML --> H[decode_bytes → clean_html_for_llm bs4 → text part + PROMPT_FOR_HTML → 1 LLM call]
|
|
197
|
+
M -- EMAIL --> E[stdlib email parse → headers + body]
|
|
198
|
+
E --> EI{inline images?}
|
|
199
|
+
EI -- yes --> EOCR[per-image OCR via LLM<br/>PROMPT_FOR_PICTURE]
|
|
200
|
+
EOCR --> ES[substitute cid placeholders]
|
|
201
|
+
EI -- no --> ES2[text only]
|
|
202
|
+
M -- DOCUMENT Office --> O[soffice convert-to pdf]
|
|
203
|
+
O --> P2[reuse PDF path]
|
|
204
|
+
```
|
|
205
|
+
|
|
206
|
+
### 6.3 AUDIO(`audio.aparse`)
|
|
207
|
+
|
|
208
|
+
```
|
|
209
|
+
RawFile.content (mp3/wav/m4a/...) → image_url data URI + PROMPT_FOR_AUDIO → 1 LLM call → ParsedContent.text
|
|
210
|
+
```
|
|
211
|
+
|
|
212
|
+
### 6.4 URL(`url.aparse`)
|
|
213
|
+
|
|
214
|
+
```mermaid
|
|
215
|
+
flowchart LR
|
|
216
|
+
A[uri or content] --> B{has content?}
|
|
217
|
+
B -- no --> C[fetch_uri httpx]
|
|
218
|
+
C -- http/https --> D[bytes + Content-Type]
|
|
219
|
+
C -- other scheme --> X[ValueError reject]
|
|
220
|
+
B -- yes --> D
|
|
221
|
+
D --> CT[get_modality_from_mime fetched Content-Type]
|
|
222
|
+
CT --> SW{inner Modality}
|
|
223
|
+
SW -- IMAGE --> IM[image.aparse]
|
|
224
|
+
SW -- AUDIO --> AU[audio.aparse]
|
|
225
|
+
SW -- PDF / DOCUMENT / EMAIL --> DC[document.aparse]
|
|
226
|
+
SW -- HTML / UNKNOWN-fallback --> HT[document.aparse extension=html]
|
|
227
|
+
HT -.OG/meta extract.-> META[og_tags / twitter_tags / favicon / ...]
|
|
228
|
+
IM --> R[ParsedContent modality=URL]
|
|
229
|
+
AU --> R
|
|
230
|
+
DC --> R
|
|
231
|
+
HT --> R
|
|
232
|
+
META --> R
|
|
233
|
+
```
|
|
234
|
+
|
|
235
|
+
**关键规则**:
|
|
236
|
+
- 抓回来的 Content-Type **驱动 inner dispatch**。`application/pdf` → PDF handler;`image/png` → image handler;`audio/mpeg` → audio handler;`text/html` 或未知 → HTML handler(兜底)。
|
|
237
|
+
- **OG / Twitter / `<meta>` 抽取仅 HTML 响应做** —— 非 HTML 响应的 `ParsedContent.metadata` 不带 `og_tags` / `title` 等字段。
|
|
238
|
+
- 返回的 `modality` 永远是 `Modality.URL`,真实的 inner 类型放在 `metadata["inner_modality"]`,调用方既能知道"这是从网络来的",也能知道"实际是 PDF"。
|
|
239
|
+
|
|
240
|
+
---
|
|
241
|
+
|
|
242
|
+
## 7. 依赖拓扑
|
|
243
|
+
|
|
244
|
+
### Pip 依赖(`pyproject.toml`)
|
|
245
|
+
|
|
246
|
+
| 依赖 | 用途 | 何时拉 |
|
|
247
|
+
|---|---|---|
|
|
248
|
+
| `everalgo-core` | LLM 客户端 + 公共类型 + fake_llm | 必装 |
|
|
249
|
+
| `asgiref` | sync 桥接(`async_to_sync`)| 必装 |
|
|
250
|
+
| `pillow` | image 转码 + 长图分块 | 必装 |
|
|
251
|
+
| `beautifulsoup4` | HTML 清洗 + URL OG 抽取 | 必装 |
|
|
252
|
+
| `httpx` | URL HTTP 抓取 | 透传(`everalgo-core` 已经依赖 openai SDK 拉了) |
|
|
253
|
+
| `cairosvg` | SVG 栅格化 | `pip install 'everalgo-parser[svg]'` 才装 |
|
|
254
|
+
|
|
255
|
+
### 系统依赖
|
|
256
|
+
|
|
257
|
+
| 程序 | 用途 | 检测 |
|
|
258
|
+
|---|---|---|
|
|
259
|
+
| `soffice`(LibreOffice)| Office 文档转 PDF | `shutil.which("soffice")` + macOS `/Applications/LibreOffice.app/Contents/MacOS/soffice` |
|
|
260
|
+
|
|
261
|
+
---
|
|
262
|
+
|
|
263
|
+
## 8. 测试覆盖
|
|
264
|
+
|
|
265
|
+
| 文件 | 数量 | 类型 | LLM |
|
|
266
|
+
|---|---|---|---|
|
|
267
|
+
| `test_public_api.py` | 2 | 公开 API 形状 | 不用 |
|
|
268
|
+
| `test_dispatch.py` | 25 | 顶层派发 | `FakeLLMClient` |
|
|
269
|
+
| `test_image.py` | 12 | PIL 路径 + 长图分块 / 合并 | `FakeLLMClient` |
|
|
270
|
+
| `test_audio.py` | 5 | audio 子模块 happy + error path | `FakeLLMClient` |
|
|
271
|
+
| `test_document.py` | 12 | PDF / HTML / EML 单测 | `FakeLLMClient` |
|
|
272
|
+
| `test_url.py` | 21 | OG 抽取 + httpx mock | `FakeLLMClient` + `respx` |
|
|
273
|
+
| `test_utils.py` | 18 | `_utils` 助手(no LLM,no network)| 不用 |
|
|
274
|
+
| `test_video.py` | 2 | video stub NotImplementedError | 不用 |
|
|
275
|
+
| `tests/types/test_modality.py` | 12 | Modality enum + get_modality | 不用 |
|
|
276
|
+
| `tests/types/test_parsed.py` | 3 | ParsedContent 字段 | 不用 |
|
|
277
|
+
| `test_e2e_all_formats.py` | 7 | 全格式 e2e | 真 LLM(env `OPENROUTER_API_KEY`)|
|
|
278
|
+
| `test_e2e_pdf_openrouter.py` | 1 | PDF spike e2e | 真 LLM(env `OPENROUTER_API_KEY`)|
|
|
279
|
+
|
|
280
|
+
**测试总数**:`391`(parser+core)/`923`(workspace 全量)
|
|
281
|
+
**E2E**(需 `OPENROUTER_API_KEY`):`8`(`test_e2e_all_formats.py` 7 + `test_e2e_pdf_openrouter.py` 1)
|
|
282
|
+
|
|
283
|
+
---
|
|
284
|
+
|
|
285
|
+
## 9. 公开规范契约
|
|
286
|
+
|
|
287
|
+
| 规范 | 对应 |
|
|
288
|
+
|---|---|
|
|
289
|
+
| `a` 前缀代表 async | ✅ `aparse` async / `parse` sync 桥接(ADR-010) |
|
|
290
|
+
| Protocol over ABC | ✅ 没有 `ABC` 基类,LLMClient 是 Protocol(ADR-011) |
|
|
291
|
+
| 无 retry / fallback / metrics | ✅ 算法层不做工程高可用(ADR-012) |
|
|
292
|
+
| 无状态(不读文件系统) | ✅ 接 `bytes` / HTTP,`file://` 拒绝(AGENTS.md §1) |
|
|
293
|
+
| 懒 `%`-format 日志 | ✅ `logger.debug("count=%d", n)` 风格(ADR-013) |
|
|
294
|
+
| Prompt = `prompts/{en,zh}/<name>.py` 模块常量 | ✅ |
|
|
295
|
+
| Numpy-style docstring | ✅ |
|
|
296
|
+
| 英文 docstring / 标识符 / commit | ✅(docs/ 下 design.md 例外,中文 OK) |
|
|
297
|
+
|
|
298
|
+
---
|
|
299
|
+
|
|
300
|
+
## 10. 已知未实现 / 推迟
|
|
301
|
+
|
|
302
|
+
| 项 | 状态 | 备注 |
|
|
303
|
+
|---|---|---|
|
|
304
|
+
| `video.py` | stub | 上游内部多模态解析库**也没有**实现,Gemini Video vs Whisper+帧抽取 选型待 ADR |
|
|
305
|
+
| `audio` BMP-like 转码 | 没做 | mp3/wav 测过;m4a/amr/aiff/aac/ogg/flac 未在 e2e 覆盖,Gemini 兼容度未全测 |
|
|
306
|
+
| `Office` 系统依赖动态检测降级 | 仅运行时报错 | 没有 import 时友好警告;`_find_soffice` 缺失就 `RuntimeError` |
|
|
307
|
+
| `T1` 主条目其他 schema | 待 RESOLVED | 我们只 RESOLVED 了 `T1.parser` 子集(`ParsedContent` / `Modality` / `RawFile`) |
|
|
@@ -4,19 +4,23 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "everalgo-parser"
|
|
7
|
-
version = "0.1
|
|
7
|
+
version = "0.2.1"
|
|
8
8
|
description = "EverAlgo parser: multimodal raw-file parsing (image / audio / document / video / url) into ParsedContent."
|
|
9
9
|
requires-python = ">=3.12"
|
|
10
|
-
license = "
|
|
10
|
+
license = "Apache-2.0"
|
|
11
11
|
readme = "README.md"
|
|
12
|
-
authors = [{ name = "EverMind" }]
|
|
12
|
+
authors = [{ name = "EverMind", email = "Evermind@shanda.com" }]
|
|
13
|
+
keywords = ["llm", "multimodal", "parsing", "everalgo"]
|
|
13
14
|
classifiers = [
|
|
14
15
|
"Development Status :: 3 - Alpha",
|
|
15
|
-
"
|
|
16
|
+
"Intended Audience :: Developers",
|
|
17
|
+
"License :: OSI Approved :: Apache Software License",
|
|
18
|
+
"Programming Language :: Python :: 3",
|
|
16
19
|
"Programming Language :: Python :: 3.12",
|
|
20
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
17
21
|
]
|
|
18
22
|
dependencies = [
|
|
19
|
-
"everalgo-core>=0.
|
|
23
|
+
"everalgo-core>=0.2.0,<2.0.0",
|
|
20
24
|
"asgiref>=3.8.0",
|
|
21
25
|
"pillow>=10.0.0",
|
|
22
26
|
"beautifulsoup4>=4.12.0",
|
|
@@ -3,7 +3,6 @@
|
|
|
3
3
|
Public surface (re-exports for ergonomic ``from everalgo.parser import ...``):
|
|
4
4
|
|
|
5
5
|
- Functions: ``aparse`` / ``parse`` — dispatch entry points.
|
|
6
|
-
- Submodules: ``image`` / ``audio`` / ``document`` / ``url`` / ``video``.
|
|
7
6
|
- Types: ``ParsedContent`` / ``Modality`` / ``RawFile`` (re-exported from
|
|
8
7
|
``everalgo.types`` for one-stop import).
|
|
9
8
|
"""
|
|
@@ -15,8 +14,9 @@ from typing import TYPE_CHECKING
|
|
|
15
14
|
from urllib.parse import urlparse
|
|
16
15
|
|
|
17
16
|
from asgiref.sync import async_to_sync
|
|
17
|
+
from PIL import Image
|
|
18
18
|
|
|
19
|
-
from everalgo.parser import audio, document, image, url
|
|
19
|
+
from everalgo.parser import audio, document, image, url
|
|
20
20
|
from everalgo.types import (
|
|
21
21
|
Modality,
|
|
22
22
|
ParsedContent,
|
|
@@ -34,17 +34,19 @@ __all__ = [
|
|
|
34
34
|
"ParsedContent",
|
|
35
35
|
"RawFile",
|
|
36
36
|
"aparse",
|
|
37
|
-
"audio",
|
|
38
|
-
"document",
|
|
39
|
-
"image",
|
|
40
37
|
"parse",
|
|
41
|
-
"url",
|
|
42
|
-
"video",
|
|
43
38
|
]
|
|
44
39
|
|
|
45
40
|
logger = logging.getLogger(__name__)
|
|
46
41
|
logger.addHandler(logging.NullHandler())
|
|
47
42
|
|
|
43
|
+
# Explicit decompression-bomb cap (Pillow default raises ``DecompressionBombError``
|
|
44
|
+
# only at ~2x this threshold). 100M pixels = 10000x10000, ample for any realistic
|
|
45
|
+
# screenshot / scan while keeping a hostile caller from exhausting memory via a
|
|
46
|
+
# crafted PNG/TIFF. Applications can override by re-assigning
|
|
47
|
+
# ``PIL.Image.MAX_IMAGE_PIXELS`` after importing the parser.
|
|
48
|
+
Image.MAX_IMAGE_PIXELS = 100_000_000
|
|
49
|
+
|
|
48
50
|
|
|
49
51
|
async def aparse(raw_file: RawFile, *, llm: LLMClient) -> ParsedContent:
|
|
50
52
|
"""Top-level dispatch: classify the input and route to a submodule.
|
|
@@ -1,16 +1,13 @@
|
|
|
1
1
|
"""Internal parser helpers.
|
|
2
2
|
|
|
3
|
-
Migrated (with English docstrings) from
|
|
4
|
-
``evermemos-multimodal/src/parser/utils.py`` (prod-20260306-0331-v1):
|
|
5
|
-
|
|
6
3
|
- ``decode_bytes`` — multi-encoding decoder for unknown-charset bytes.
|
|
7
4
|
- ``check_aspect_ratio`` / ``split_image_with_overlap`` — long-image (tall
|
|
8
5
|
screenshot) handling for the image parser.
|
|
9
6
|
- ``clean_html_content`` — strict text extraction (drops all tags).
|
|
10
7
|
- ``clean_html_for_llm`` — preserves structural tags inside ``<body>``,
|
|
11
8
|
strips known-noise tags and useless attributes, returns HTML.
|
|
12
|
-
- ``fetch_uri`` — HTTP(S) fetch for the URL parser. Adapted from
|
|
13
|
-
|
|
9
|
+
- ``fetch_uri`` — HTTP(S) fetch for the URL parser. Adapted from an
|
|
10
|
+
upstream internal URL extractor reference implementation.
|
|
14
11
|
"""
|
|
15
12
|
|
|
16
13
|
from __future__ import annotations
|
|
@@ -43,7 +40,7 @@ _DEFAULT_USER_AGENT = "Mozilla/5.0 (compatible; everalgo-parser/0.1; +https://ev
|
|
|
43
40
|
async def fetch_uri(uri: str, *, timeout: float = 30.0) -> tuple[bytes, str]:
|
|
44
41
|
"""Fetch ``http``/``https`` URI and return ``(content_bytes, content_type)``.
|
|
45
42
|
|
|
46
|
-
Adapted from
|
|
43
|
+
Adapted from an upstream internal URL extractor reference implementation;
|
|
47
44
|
rejects non-http(s) schemes per AGENTS.md §1 (no filesystem access).
|
|
48
45
|
|
|
49
46
|
Parameters
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
"""Audio parser — ASR via multimodal LLM.
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
audio bytes as a single ``image_url`` data URI (OpenRouter / Gemini accepts
|
|
3
|
+
Sends the audio bytes as a single ``image_url`` data URI (OpenRouter / Gemini accepts
|
|
5
4
|
audio MIMEs through the same content-part shape) and the LLM transcribes.
|
|
6
5
|
"""
|
|
7
6
|
|
|
@@ -4,8 +4,7 @@ Per ``AGENTS.md`` and the engineering alignment for the parser migration, the
|
|
|
4
4
|
``document`` submodule covers every non-image / non-audio file shape:
|
|
5
5
|
|
|
6
6
|
- ``Modality.PDF`` — sent as a single ``application/pdf`` data URI to the
|
|
7
|
-
multimodal LLM
|
|
8
|
-
``evermemos-multimodal/src/parser/pdf/gemini_parser.py``).
|
|
7
|
+
multimodal LLM.
|
|
9
8
|
- ``Modality.HTML`` — bs4-cleaned (``_utils.clean_html_for_llm``) before
|
|
10
9
|
sending the structured HTML to the LLM with the HTML prompt.
|
|
11
10
|
- ``Modality.EMAIL`` — pure-python ``email`` stdlib parsing; when inline
|
|
@@ -50,8 +49,7 @@ _PDF_MIME = "application/pdf"
|
|
|
50
49
|
_HTML_MAX_INPUT_CHARS = 1_000_000
|
|
51
50
|
_MACOS_SOFFICE = "/Applications/LibreOffice.app/Contents/MacOS/soffice"
|
|
52
51
|
|
|
53
|
-
#
|
|
54
|
-
# inline image OCR caps at 8K tokens (single-image scope, no need for the
|
|
52
|
+
# Inline image OCR caps at 8K tokens (single-image scope, no need for the
|
|
55
53
|
# 32K merge budget used by the long-image stitcher).
|
|
56
54
|
_INLINE_OCR_MAX_TOKENS = 8_000
|
|
57
55
|
_INLINE_IMAGE_MIMETYPES = frozenset(
|
|
@@ -160,8 +158,6 @@ async def _aparse_html(raw_file: RawFile, *, llm: LLMClient) -> ParsedContent:
|
|
|
160
158
|
async def _aparse_email(raw_file: RawFile, *, llm: LLMClient) -> ParsedContent:
|
|
161
159
|
"""EML → ``ParsedContent`` via stdlib ``email`` + inline-image OCR.
|
|
162
160
|
|
|
163
|
-
Mirrors evermemos-multimodal/src/parser/email/eml_parser.py:
|
|
164
|
-
|
|
165
161
|
1. Extract headers.
|
|
166
162
|
2. Collect inline images (Content-ID → bytes).
|
|
167
163
|
3. Extract body: prefer ``text/plain`` when there are no cid images
|
|
@@ -261,7 +257,7 @@ async def _aparse_office(
|
|
|
261
257
|
|
|
262
258
|
|
|
263
259
|
def _extract_email_headers(msg: EmailMessage) -> str:
|
|
264
|
-
lines = []
|
|
260
|
+
lines: list[str] = []
|
|
265
261
|
for field in ("Subject", "From", "To", "Cc", "Date"):
|
|
266
262
|
value = msg.get(field)
|
|
267
263
|
if value:
|
|
@@ -367,7 +363,7 @@ async def _ocr_inline_images(
|
|
|
367
363
|
except Exception as exc:
|
|
368
364
|
# Best-effort per-image: a single OCR failure should not abort the
|
|
369
365
|
# whole email parse — log and continue with a placeholder marker.
|
|
370
|
-
logger.warning("OCR failed for inline image %s: %s", filename, exc)
|
|
366
|
+
logger.warning("OCR failed for inline image %s: %s", filename, exc, exc_info=True)
|
|
371
367
|
results[idx] = f"[image: {filename}, OCR failed]"
|
|
372
368
|
return results
|
|
373
369
|
|
|
@@ -403,6 +399,11 @@ async def _convert_to_pdf_via_soffice(
|
|
|
403
399
|
content: bytes, extension: str, soffice: str
|
|
404
400
|
) -> bytes: # pragma: no cover (libreoffice binary, not in CI)
|
|
405
401
|
"""Run LibreOffice headless to convert bytes → PDF bytes."""
|
|
402
|
+
# Reject path-traversal / directory separators in `extension` so a hostile
|
|
403
|
+
# caller cannot escape the tempdir via e.g. ``extension="../../etc/passwd"``.
|
|
404
|
+
# `extension` is expected to be a short ASCII suffix (docx / xlsx / ...).
|
|
405
|
+
if not extension or not extension.isalnum() or len(extension) > 8:
|
|
406
|
+
raise ValueError(f"invalid extension for soffice conversion: {extension!r}")
|
|
406
407
|
with tempfile.TemporaryDirectory() as tmpdir: # pragma: no cover
|
|
407
408
|
in_path = Path(tmpdir) / f"input.{extension}" # pragma: no cover
|
|
408
409
|
in_path.write_bytes(content) # pragma: no cover
|
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
"""Image parser — multimodal LLM OCR with tall-image splitting.
|
|
2
2
|
|
|
3
|
-
Mirrors ``evermemos-multimodal/src/parser/image/gemini_parser.py``:
|
|
4
|
-
|
|
5
3
|
- Normal-ratio images go through a single LLM call.
|
|
6
4
|
- Tall screenshots (height/width > 10) are split into overlapping slices
|
|
7
5
|
via ``_utils.split_image_with_overlap``; each slice is OCR'd independently
|
|
@@ -54,10 +52,8 @@ _SVG_EXTENSIONS: frozenset[str] = frozenset({"svg"})
|
|
|
54
52
|
|
|
55
53
|
_TALL_IMAGE_RATIO_THRESHOLD = 10.0
|
|
56
54
|
|
|
57
|
-
# Token budgets
|
|
58
|
-
#
|
|
59
|
-
# caps at 8K; the merge step gets 4x because it joins multiple OCR'd chunks
|
|
60
|
-
# and the merged text is longer than any individual slice.
|
|
55
|
+
# Token budgets. Per-slice OCR caps at 8K; the merge step gets 4x because it
|
|
56
|
+
# joins multiple OCR'd chunks and the merged text is longer than any individual slice.
|
|
61
57
|
_MAX_OCR_OUTPUT_TOKENS = 8_000
|
|
62
58
|
_MAX_MERGE_OUTPUT_TOKENS = 32_000
|
|
63
59
|
|
|
@@ -138,9 +134,6 @@ async def _aparse_tall(
|
|
|
138
134
|
) -> ParsedContent:
|
|
139
135
|
r"""Split-then-merge OCR for tall screenshots.
|
|
140
136
|
|
|
141
|
-
Mirrors ``evermemos-multimodal/src/parser/image/base.py:parse`` +
|
|
142
|
-
``gemini_parser._merge_parts``:
|
|
143
|
-
|
|
144
137
|
1. Split into N overlapping vertical slices.
|
|
145
138
|
2. OCR each slice; **drop empty results** so the merge step sees only
|
|
146
139
|
real text (an all-empty slice from a blank PDF region would otherwise
|
|
@@ -223,7 +216,9 @@ async def _aparse_tall(
|
|
|
223
216
|
merge_fallback_used = True
|
|
224
217
|
last_response = merge_resp
|
|
225
218
|
except Exception as exc:
|
|
226
|
-
logger.warning(
|
|
219
|
+
logger.warning(
|
|
220
|
+
"LLM merge of tall-image OCR parts failed (%s); falling back to deterministic join", exc, exc_info=True
|
|
221
|
+
)
|
|
227
222
|
merged_text = "\n\n".join(ocr_results)
|
|
228
223
|
merge_fallback_used = True
|
|
229
224
|
|
|
@@ -1,8 +1,4 @@
|
|
|
1
|
-
"""Audio parser prompt.
|
|
2
|
-
|
|
3
|
-
Lifted verbatim from ``evermemos-multimodal/src/parser/prompts.py``
|
|
4
|
-
(prod-20260306-0331-v1).
|
|
5
|
-
"""
|
|
1
|
+
"""Audio parser prompt."""
|
|
6
2
|
|
|
7
3
|
PROMPT_FOR_AUDIO = """This is a voice message sent by a user in a chat application. Transcribe it word by word.
|
|
8
4
|
|
|
@@ -1,9 +1,4 @@
|
|
|
1
|
-
"""Document parser prompts (PDF / Office / HTML).
|
|
2
|
-
|
|
3
|
-
Lifted verbatim from ``evermemos-multimodal/src/parser/prompts.py``
|
|
4
|
-
(prod-20260306-0331-v1) where these strings have been iterated on against
|
|
5
|
-
real customer documents.
|
|
6
|
-
"""
|
|
1
|
+
"""Document parser prompts (PDF / Office / HTML)."""
|
|
7
2
|
|
|
8
3
|
PROMPT_FOR_FILE = """Read this document and return its content as if you were reading it naturally.
|
|
9
4
|
|
|
@@ -1,8 +1,4 @@
|
|
|
1
|
-
"""Image parser prompts.
|
|
2
|
-
|
|
3
|
-
Lifted verbatim from ``evermemos-multimodal/src/parser/prompts.py``
|
|
4
|
-
(prod-20260306-0331-v1).
|
|
5
|
-
"""
|
|
1
|
+
"""Image parser prompts."""
|
|
6
2
|
|
|
7
3
|
PROMPT_FOR_PICTURE = """Read this image and return its content as if you were reading it naturally.
|
|
8
4
|
|
|
@@ -18,10 +18,10 @@ The returned ``ParsedContent.modality`` is always ``Modality.URL`` so
|
|
|
18
18
|
callers can distinguish "loaded from network" from "passed in as bytes";
|
|
19
19
|
the inner modality is recorded in ``metadata["inner_modality"]``.
|
|
20
20
|
|
|
21
|
-
Schema source for the metadata extractor:
|
|
22
|
-
``
|
|
23
|
-
|
|
24
|
-
``
|
|
21
|
+
Schema source for the metadata extractor: an upstream internal URL
|
|
22
|
+
extractor reference implementation (``_extract_og_tags`` /
|
|
23
|
+
``_extract_twitter_tags`` / ``_extract_meta_tags`` / ``_extract_title`` /
|
|
24
|
+
``_extract_favicon``).
|
|
25
25
|
"""
|
|
26
26
|
|
|
27
27
|
from __future__ import annotations
|
|
@@ -166,7 +166,7 @@ async def _dispatch_inner(
|
|
|
166
166
|
raise ValueError(f"url.aparse: cannot dispatch fetched body for modality={modality.value!r}")
|
|
167
167
|
|
|
168
168
|
|
|
169
|
-
# ---- metadata extraction (port from
|
|
169
|
+
# ---- metadata extraction (port from upstream internal URL extractor) ----
|
|
170
170
|
|
|
171
171
|
|
|
172
172
|
def _extract_metadata(html_text: str, *, base_url: str) -> dict[str, object]:
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""Video parser — deferred.
|
|
2
|
+
|
|
3
|
+
No implementation yet; selection between Gemini Video and Whisper + frame
|
|
4
|
+
extraction is a follow-up decision. Calling ``aparse`` / ``parse`` today
|
|
5
|
+
raises ``NotImplementedError`` so dispatch in ``everalgo.parser.aparse``
|
|
6
|
+
surfaces the deferred status cleanly to callers.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from typing import TYPE_CHECKING
|
|
12
|
+
|
|
13
|
+
if TYPE_CHECKING:
|
|
14
|
+
from everalgo.llm import LLMClient
|
|
15
|
+
from everalgo.types import ParsedContent, RawFile
|
|
16
|
+
|
|
17
|
+
__all__ = ["aparse", "parse"]
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
async def aparse(raw_file: RawFile, *, llm: LLMClient) -> ParsedContent:
|
|
21
|
+
"""Parse video → ``ParsedContent``. Deferred — see module docstring."""
|
|
22
|
+
raise NotImplementedError(
|
|
23
|
+
"video parser is deferred; selection between Gemini Video / Whisper + frame extraction pending an ADR."
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def parse(raw_file: RawFile, *, llm: LLMClient) -> ParsedContent:
|
|
28
|
+
"""Sync bridge. Deferred — see module docstring."""
|
|
29
|
+
raise NotImplementedError(
|
|
30
|
+
"video parser is deferred; selection between Gemini Video / Whisper + frame extraction pending an ADR."
|
|
31
|
+
)
|
|
@@ -246,7 +246,7 @@ async def test_aparse_raises_for_unsupported_modality(monkeypatch: pytest.Monkey
|
|
|
246
246
|
from everalgo.types import Modality
|
|
247
247
|
|
|
248
248
|
# Modality.URL is valid but not handled by the dispatch switch (IMAGE/AUDIO/PDF/etc.)
|
|
249
|
-
monkeypatch.setattr(pkg, "get_modality", lambda _ext: Modality.URL)
|
|
249
|
+
monkeypatch.setattr(pkg, "get_modality", lambda _ext: Modality.URL) # type: ignore[misc]
|
|
250
250
|
with pytest.raises(ValueError, match="Unsupported modality"):
|
|
251
251
|
await parser_pkg.aparse(RawFile(content=b"x", extension="mp4"), llm=_fake())
|
|
252
252
|
|
|
@@ -130,7 +130,7 @@ async def test_aparse_html_empty_after_clean_returns_empty_parsed_content(monkey
|
|
|
130
130
|
import everalgo.parser.document as doc_mod
|
|
131
131
|
|
|
132
132
|
# Monkeypatch clean_html_for_llm to return "" so the empty-after-cleanup branch fires
|
|
133
|
-
monkeypatch.setattr(doc_mod, "clean_html_for_llm", lambda _: "")
|
|
133
|
+
monkeypatch.setattr(doc_mod, "clean_html_for_llm", lambda _: "") # type: ignore[misc]
|
|
134
134
|
|
|
135
135
|
fake = _make_fake()
|
|
136
136
|
html = b"<html><body><p>some content</p></body></html>"
|
|
@@ -10,16 +10,8 @@ def test_top_level_routing_aparse_parse() -> None:
|
|
|
10
10
|
assert callable(parse)
|
|
11
11
|
|
|
12
12
|
|
|
13
|
-
def test_five_submodules_have_dual_interface() -> None:
|
|
14
|
-
from everalgo.parser import audio, document, image, url, video
|
|
15
|
-
|
|
16
|
-
for mod in (image, audio, document, video, url):
|
|
17
|
-
assert inspect.iscoroutinefunction(mod.aparse)
|
|
18
|
-
assert callable(mod.parse)
|
|
19
|
-
|
|
20
|
-
|
|
21
13
|
def test_dunder_all_lists_public_symbols() -> None:
|
|
22
|
-
"""
|
|
14
|
+
"""``__all__`` lists only facade-level symbols; submodules are internal."""
|
|
23
15
|
from everalgo.parser import __all__
|
|
24
16
|
|
|
25
17
|
assert sorted(__all__) == sorted(
|
|
@@ -28,11 +20,6 @@ def test_dunder_all_lists_public_symbols() -> None:
|
|
|
28
20
|
"ParsedContent",
|
|
29
21
|
"RawFile",
|
|
30
22
|
"aparse",
|
|
31
|
-
"audio",
|
|
32
|
-
"document",
|
|
33
|
-
"image",
|
|
34
23
|
"parse",
|
|
35
|
-
"url",
|
|
36
|
-
"video",
|
|
37
24
|
]
|
|
38
25
|
)
|
|
@@ -27,26 +27,26 @@ def _png_bytes(width: int, height: int) -> bytes:
|
|
|
27
27
|
def test_check_aspect_ratio_normal_image_not_tall() -> None:
|
|
28
28
|
needs_split, ratio = check_aspect_ratio(_png_bytes(100, 200))
|
|
29
29
|
assert needs_split is False
|
|
30
|
-
assert ratio == pytest.approx(2.0)
|
|
30
|
+
assert ratio == pytest.approx(2.0) # pyright: ignore[reportUnknownMemberType]
|
|
31
31
|
|
|
32
32
|
|
|
33
33
|
def test_check_aspect_ratio_tall_image_needs_split() -> None:
|
|
34
34
|
needs_split, ratio = check_aspect_ratio(_png_bytes(100, 2000))
|
|
35
35
|
assert needs_split is True
|
|
36
|
-
assert ratio == pytest.approx(20.0)
|
|
36
|
+
assert ratio == pytest.approx(20.0) # pyright: ignore[reportUnknownMemberType]
|
|
37
37
|
|
|
38
38
|
|
|
39
39
|
def test_check_aspect_ratio_custom_max_ratio() -> None:
|
|
40
40
|
# 200 / 100 = 2.0; with max_ratio=1.5 it should need split
|
|
41
41
|
needs_split, ratio = check_aspect_ratio(_png_bytes(100, 200), max_ratio=1.5)
|
|
42
42
|
assert needs_split is True
|
|
43
|
-
assert ratio == pytest.approx(2.0)
|
|
43
|
+
assert ratio == pytest.approx(2.0) # pyright: ignore[reportUnknownMemberType]
|
|
44
44
|
|
|
45
45
|
|
|
46
46
|
def test_check_aspect_ratio_square_image() -> None:
|
|
47
47
|
needs_split, ratio = check_aspect_ratio(_png_bytes(100, 100))
|
|
48
48
|
assert needs_split is False
|
|
49
|
-
assert ratio == pytest.approx(1.0)
|
|
49
|
+
assert ratio == pytest.approx(1.0) # pyright: ignore[reportUnknownMemberType]
|
|
50
50
|
|
|
51
51
|
|
|
52
52
|
# ---- split_image_with_overlap ----
|
|
@@ -1,38 +0,0 @@
|
|
|
1
|
-
"""Video parser — deferred.
|
|
2
|
-
|
|
3
|
-
evermemos-multimodal has no video parser implementation; the upstream
|
|
4
|
-
``FILE_TYPE_DEFAULT_EXTENSION`` map reserves ``"video": "mp4"`` but no
|
|
5
|
-
parser ever consumed it. Selection between Gemini Video and Whisper +
|
|
6
|
-
frame extraction is a follow-up decision; when that lands, this stub is
|
|
7
|
-
replaced with the real handler. Calling ``aparse`` / ``parse`` today raises
|
|
8
|
-
``NotImplementedError`` so dispatch in ``everalgo.parser.aparse`` surfaces
|
|
9
|
-
the deferred status cleanly to callers.
|
|
10
|
-
"""
|
|
11
|
-
|
|
12
|
-
from __future__ import annotations
|
|
13
|
-
|
|
14
|
-
from typing import TYPE_CHECKING
|
|
15
|
-
|
|
16
|
-
if TYPE_CHECKING:
|
|
17
|
-
from everalgo.llm import LLMClient
|
|
18
|
-
from everalgo.types import ParsedContent, RawFile
|
|
19
|
-
|
|
20
|
-
__all__ = ["aparse", "parse"]
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
async def aparse(raw_file: RawFile, *, llm: LLMClient) -> ParsedContent:
|
|
24
|
-
"""Parse video → ``ParsedContent``. Deferred — see module docstring."""
|
|
25
|
-
raise NotImplementedError(
|
|
26
|
-
"video parser is deferred; selection between Gemini Video / Whisper "
|
|
27
|
-
"+ frame extraction pending an ADR. No upstream implementation exists "
|
|
28
|
-
"in evermemos-multimodal to port."
|
|
29
|
-
)
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
def parse(raw_file: RawFile, *, llm: LLMClient) -> ParsedContent:
|
|
33
|
-
"""Sync bridge. Deferred — see module docstring."""
|
|
34
|
-
raise NotImplementedError(
|
|
35
|
-
"video parser is deferred; selection between Gemini Video / Whisper "
|
|
36
|
-
"+ frame extraction pending an ADR. No upstream implementation exists "
|
|
37
|
-
"in evermemos-multimodal to port."
|
|
38
|
-
)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|