viparse 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- viparse-0.1.0/.claude/rules/code-style.md +21 -0
- viparse-0.1.0/.claude/rules/commit-title.md +29 -0
- viparse-0.1.0/.claude/rules/review-before-push.md +19 -0
- viparse-0.1.0/.claude/skills/add-encoding/SKILL.md +53 -0
- viparse-0.1.0/.claude/skills/add-engine/SKILL.md +54 -0
- viparse-0.1.0/.claude/skills/release/SKILL.md +56 -0
- viparse-0.1.0/.github/dependabot.yml +17 -0
- viparse-0.1.0/.github/pull_request_template.md +13 -0
- viparse-0.1.0/.github/workflows/ci.yml +90 -0
- viparse-0.1.0/.github/workflows/publish.yml +32 -0
- viparse-0.1.0/.gitignore +25 -0
- viparse-0.1.0/.pre-commit-config.yaml +32 -0
- viparse-0.1.0/CHANGELOG.md +44 -0
- viparse-0.1.0/CLAUDE.md +90 -0
- viparse-0.1.0/CONTRIBUTING.md +29 -0
- viparse-0.1.0/PKG-INFO +83 -0
- viparse-0.1.0/README.md +39 -0
- viparse-0.1.0/SECURITY.md +35 -0
- viparse-0.1.0/docs/examples/README.md +14 -0
- viparse-0.1.0/docs/examples/rag_pipeline.py +57 -0
- viparse-0.1.0/docs/ops/linear-github-mapping.md +52 -0
- viparse-0.1.0/docs/specs/README.md +64 -0
- viparse-0.1.0/docs/specs/SPEC-0-foundation-ops.md +80 -0
- viparse-0.1.0/docs/specs/SPEC-1-core-architecture.md +92 -0
- viparse-0.1.0/docs/specs/SPEC-2-extraction-engines.md +73 -0
- viparse-0.1.0/docs/specs/SPEC-3-vietnamese-normalization.md +75 -0
- viparse-0.1.0/docs/specs/SPEC-4-rag-output.md +63 -0
- viparse-0.1.0/docs/specs/SPEC-5-api-cli-packaging.md +55 -0
- viparse-0.1.0/docs/specs/SPEC-6-testing-quality.md +62 -0
- viparse-0.1.0/docs/specs/SPEC-7-performance-scale.md +57 -0
- viparse-0.1.0/docs/specs/SPEC-8-security-supplychain.md +63 -0
- viparse-0.1.0/pyproject.toml +75 -0
- viparse-0.1.0/scripts/check_nfc.py +61 -0
- viparse-0.1.0/scripts/dev.sh +83 -0
- viparse-0.1.0/src/viparse/__init__.py +106 -0
- viparse-0.1.0/src/viparse/api.py +250 -0
- viparse-0.1.0/src/viparse/cache.py +120 -0
- viparse-0.1.0/src/viparse/chunk.py +128 -0
- viparse-0.1.0/src/viparse/cli.py +203 -0
- viparse-0.1.0/src/viparse/config.py +175 -0
- viparse-0.1.0/src/viparse/detect.py +107 -0
- viparse-0.1.0/src/viparse/engines/__init__.py +6 -0
- viparse-0.1.0/src/viparse/engines/_shared.py +25 -0
- viparse-0.1.0/src/viparse/engines/docx.py +164 -0
- viparse-0.1.0/src/viparse/engines/legacy.py +136 -0
- viparse-0.1.0/src/viparse/engines/ocr.py +125 -0
- viparse-0.1.0/src/viparse/engines/pdf.py +113 -0
- viparse-0.1.0/src/viparse/engines/xlsx.py +154 -0
- viparse-0.1.0/src/viparse/errors.py +53 -0
- viparse-0.1.0/src/viparse/integrations/__init__.py +12 -0
- viparse-0.1.0/src/viparse/integrations/_common.py +51 -0
- viparse-0.1.0/src/viparse/integrations/langchain.py +38 -0
- viparse-0.1.0/src/viparse/integrations/llamaindex.py +37 -0
- viparse-0.1.0/src/viparse/model.py +178 -0
- viparse-0.1.0/src/viparse/normalize/__init__.py +7 -0
- viparse-0.1.0/src/viparse/normalize/cleanup.py +52 -0
- viparse-0.1.0/src/viparse/normalize/detector.py +131 -0
- viparse-0.1.0/src/viparse/normalize/encodings.py +30 -0
- viparse-0.1.0/src/viparse/normalize/frequency.py +60 -0
- viparse-0.1.0/src/viparse/normalize/normalizer.py +137 -0
- viparse-0.1.0/src/viparse/normalize/tables.py +91 -0
- viparse-0.1.0/src/viparse/normalize/tcvn3.py +44 -0
- viparse-0.1.0/src/viparse/normalize/viscii.py +147 -0
- viparse-0.1.0/src/viparse/normalize/vni.py +37 -0
- viparse-0.1.0/src/viparse/observability.py +39 -0
- viparse-0.1.0/src/viparse/options.py +60 -0
- viparse-0.1.0/src/viparse/pipeline.py +238 -0
- viparse-0.1.0/src/viparse/protocols.py +88 -0
- viparse-0.1.0/src/viparse/registry.py +51 -0
- viparse-0.1.0/src/viparse/safety.py +61 -0
- viparse-0.1.0/src/viparse/structure/__init__.py +7 -0
- viparse-0.1.0/src/viparse/structure/renderer.py +130 -0
- viparse-0.1.0/tests/conftest.py +30 -0
- viparse-0.1.0/tests/golden/docx_basic.json +38 -0
- viparse-0.1.0/tests/golden/docx_basic.md +7 -0
- viparse-0.1.0/tests/golden/docx_legacy.json +18 -0
- viparse-0.1.0/tests/golden/docx_legacy.md +1 -0
- viparse-0.1.0/tests/golden/pdf_basic.json +31 -0
- viparse-0.1.0/tests/golden/pdf_basic.md +5 -0
- viparse-0.1.0/tests/golden/xlsx_basic.json +32 -0
- viparse-0.1.0/tests/golden/xlsx_basic.md +5 -0
- viparse-0.1.0/tests/test_api.py +147 -0
- viparse-0.1.0/tests/test_cache.py +86 -0
- viparse-0.1.0/tests/test_chunk.py +190 -0
- viparse-0.1.0/tests/test_cli.py +225 -0
- viparse-0.1.0/tests/test_config.py +228 -0
- viparse-0.1.0/tests/test_detect.py +106 -0
- viparse-0.1.0/tests/test_encoding_tables.py +53 -0
- viparse-0.1.0/tests/test_engines_docx.py +187 -0
- viparse-0.1.0/tests/test_engines_legacy.py +165 -0
- viparse-0.1.0/tests/test_engines_ocr.py +153 -0
- viparse-0.1.0/tests/test_engines_pdf.py +156 -0
- viparse-0.1.0/tests/test_engines_xlsx.py +172 -0
- viparse-0.1.0/tests/test_errors.py +42 -0
- viparse-0.1.0/tests/test_golden_snapshots.py +118 -0
- viparse-0.1.0/tests/test_integrations.py +147 -0
- viparse-0.1.0/tests/test_model.py +128 -0
- viparse-0.1.0/tests/test_mvp_docx_normalize.py +30 -0
- viparse-0.1.0/tests/test_normalize_cleanup.py +79 -0
- viparse-0.1.0/tests/test_normalize_detector.py +66 -0
- viparse-0.1.0/tests/test_normalize_frequency.py +75 -0
- viparse-0.1.0/tests/test_normalize_tables.py +115 -0
- viparse-0.1.0/tests/test_normalize_viscii.py +64 -0
- viparse-0.1.0/tests/test_normalizer.py +253 -0
- viparse-0.1.0/tests/test_observability.py +149 -0
- viparse-0.1.0/tests/test_packaging.py +55 -0
- viparse-0.1.0/tests/test_pipeline.py +271 -0
- viparse-0.1.0/tests/test_protocols.py +99 -0
- viparse-0.1.0/tests/test_registry.py +72 -0
- viparse-0.1.0/tests/test_safety.py +55 -0
- viparse-0.1.0/tests/test_smoke.py +8 -0
- viparse-0.1.0/tests/test_structure_renderer.py +149 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# Rule: code style
|
|
2
|
+
|
|
3
|
+
Python code in viparse is linted, formatted, and type-checked automatically. The tools are the
|
|
4
|
+
source of truth; this rule states the intent behind them.
|
|
5
|
+
|
|
6
|
+
## Tooling (enforced by CI and pre-commit)
|
|
7
|
+
|
|
8
|
+
- **ruff** for both lint and format — do not hand-format around it. Run `scripts/dev.sh lint`.
|
|
9
|
+
- **mypy strict** — `scripts/dev.sh type` must report no errors. Do not weaken strictness or add
|
|
10
|
+
blanket `# type: ignore`; if an ignore is unavoidable, make it specific (`# type: ignore[code]`)
|
|
11
|
+
and comment why.
|
|
12
|
+
|
|
13
|
+
## Conventions
|
|
14
|
+
|
|
15
|
+
- **Type hints are mandatory.** Every function signature (parameters and return) is annotated.
|
|
16
|
+
Prefer precise types over `Any`; reach for `typing`/`collections.abc` generics.
|
|
17
|
+
- **English docstrings.** Public modules, classes, and functions have a short docstring, in English,
|
|
18
|
+
describing behavior — not restating the signature.
|
|
19
|
+
- **English everywhere.** All identifiers, comments, and docstrings are in English. Vietnamese
|
|
20
|
+
belongs only in test fixtures and data, never in code or prose.
|
|
21
|
+
- **NFC.** Any string literals or fixtures containing Vietnamese text are Unicode NFC.
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# Rule: commit & PR titles
|
|
2
|
+
|
|
3
|
+
Every task lands as **one commit = one PR** with a single-line title and **no description body**.
|
|
4
|
+
|
|
5
|
+
## Format
|
|
6
|
+
|
|
7
|
+
```
|
|
8
|
+
VIP-<id> <short imperative>
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
- Start with the Linear task id (`VIP-<id>`), then a short imperative summary.
|
|
12
|
+
- Keep the whole title **≤ 60 characters**.
|
|
13
|
+
- Imperative mood, capitalized, no trailing period. Write "Add", not "Added"/"Adds".
|
|
14
|
+
- **No PR description.** The PR template checklist is the review contract; a body is not required.
|
|
15
|
+
- **No `Co-Authored-By` trailer.**
|
|
16
|
+
|
|
17
|
+
## Examples
|
|
18
|
+
|
|
19
|
+
```
|
|
20
|
+
VIP-31 Add minimal PR template
|
|
21
|
+
VIP-33 Add scripts/dev.sh
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
Not:
|
|
25
|
+
|
|
26
|
+
```
|
|
27
|
+
Added a PR template. # past tense, no VIP id
|
|
28
|
+
VIP-31: add minimal pull request template for the repository # too long, colon, lowercase
|
|
29
|
+
```
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# Rule: review before push
|
|
2
|
+
|
|
3
|
+
Before pushing a branch and opening its PR, verify every item below. If a task involves real
|
|
4
|
+
code or logic, also run `/code-review` on the working changes and resolve its findings first.
|
|
5
|
+
|
|
6
|
+
## Checklist
|
|
7
|
+
|
|
8
|
+
- [ ] **Tests green** — `scripts/dev.sh test` passes locally.
|
|
9
|
+
- [ ] **Type-check clean** — `scripts/dev.sh type` (mypy strict) reports no errors.
|
|
10
|
+
- [ ] **Lint & format** — `scripts/dev.sh lint` passes (ruff check + format).
|
|
11
|
+
- [ ] **Coverage gate met** — coverage stays at or above the `fail_under` threshold.
|
|
12
|
+
- [ ] **No sensitive fixtures** — no secrets, credentials, or private/real documents committed;
|
|
13
|
+
test fixtures are synthetic or properly licensed.
|
|
14
|
+
- [ ] **NFC-safe** — any emitted or committed text is Unicode NFC (the pre-commit `nfc-check`
|
|
15
|
+
hook must pass).
|
|
16
|
+
- [ ] **Scope** — the diff matches exactly one `VIP-<id>`; nothing unrelated is bundled in.
|
|
17
|
+
|
|
18
|
+
Running `scripts/dev.sh` (no argument) covers lint + type-check + test in one go — the same gates
|
|
19
|
+
CI enforces.
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: add-encoding
|
|
3
|
+
description: Scaffold a legacy Vietnamese encoding table (e.g. TCVN3, VNI, VISCII) plus its golden test, so the normalizer can convert that encoding to Unicode NFC. Use when adding or correcting support for a legacy font/encoding.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Add a legacy encoding
|
|
7
|
+
|
|
8
|
+
Legacy Vietnamese documents store text in font-specific encodings (TCVN3 `.VnTime`, VNI
|
|
9
|
+
`VNI-Times`, VISCII, …). The normalization layer maps those bytes/codepoints to correct Unicode
|
|
10
|
+
and enforces **NFC**. This is the product's moat, so correctness is verified by **golden tests**.
|
|
11
|
+
|
|
12
|
+
Follow these steps.
|
|
13
|
+
|
|
14
|
+
## 1. Gather an authoritative mapping
|
|
15
|
+
|
|
16
|
+
- Find a reputable source for the encoding→Unicode mapping (published charset tables, the font's
|
|
17
|
+
documented layout). Record the source in a comment.
|
|
18
|
+
- Note how the encoding is detected: font name signal from the engine, byte-range heuristics, or an
|
|
19
|
+
explicit caller hint.
|
|
20
|
+
|
|
21
|
+
## 2. Add the table
|
|
22
|
+
|
|
23
|
+
Create `src/viparse/normalize/tables/<encoding>.py` (or the project's table location) as a plain,
|
|
24
|
+
reviewable mapping. Keep it data, not logic — the shared conversion routine consumes any table.
|
|
25
|
+
|
|
26
|
+
- Map every codepoint the encoding uses, including composed diacritic forms.
|
|
27
|
+
- The conversion output must be normalized to **NFC** by the shared normalizer, not per-table.
|
|
28
|
+
|
|
29
|
+
## 3. Wire detection
|
|
30
|
+
|
|
31
|
+
Register the encoding so the normalizer selects it from the engine's raw signals (e.g. font name).
|
|
32
|
+
Do not guess from the file extension.
|
|
33
|
+
|
|
34
|
+
## 4. Add a golden test
|
|
35
|
+
|
|
36
|
+
- Add an input fixture in the legacy encoding and an expected Unicode **NFC** output fixture under
|
|
37
|
+
`tests/` (synthetic text — no sensitive documents).
|
|
38
|
+
- Assert the converted output equals the golden output **and** that it is NFC
|
|
39
|
+
(`unicodedata.is_normalized("NFC", out)`).
|
|
40
|
+
- Include diacritic-heavy words (e.g. "Việt", "Nguyễn", "Đường") to catch composition bugs.
|
|
41
|
+
|
|
42
|
+
## 5. Verify
|
|
43
|
+
|
|
44
|
+
Run `scripts/dev.sh`. Open one PR titled `VIP-<id> Add <encoding> encoding`.
|
|
45
|
+
|
|
46
|
+
## Checklist
|
|
47
|
+
|
|
48
|
+
- [ ] Mapping sourced from an authoritative reference (cited in a comment)
|
|
49
|
+
- [ ] Table is data-only; conversion reuses the shared routine
|
|
50
|
+
- [ ] Output enforced to Unicode NFC
|
|
51
|
+
- [ ] Detection wired from engine signals, not the extension
|
|
52
|
+
- [ ] Golden test with diacritic-heavy input asserts exact NFC output
|
|
53
|
+
- [ ] `scripts/dev.sh` passes
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: add-engine
|
|
3
|
+
description: Scaffold a new viparse extraction Engine — a thin adapter around a well-maintained parse library, plus its test and registry entry. Use when adding support for a new file format or swapping the library behind an existing format.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Add an extraction engine
|
|
7
|
+
|
|
8
|
+
An **Engine** is a thin adapter that wraps a well-maintained parse library and returns a
|
|
9
|
+
`RawExtraction` (raw text + font/encoding signals). Engines carry **no Vietnamese logic** — that
|
|
10
|
+
lives in the normalization layer. Heavy engines are lazy-imported behind an extra.
|
|
11
|
+
|
|
12
|
+
Follow these steps.
|
|
13
|
+
|
|
14
|
+
## 1. Confirm the library choice
|
|
15
|
+
|
|
16
|
+
- Pick a **well-maintained** library for the format (recent releases, no open CVEs, active repo).
|
|
17
|
+
Never hand-write the parser.
|
|
18
|
+
- Decide whether it is heavy (native deps, OCR, LibreOffice). If so, it goes behind an extra
|
|
19
|
+
(`viparse[<extra>]`) and must be **lazy-imported** inside the engine, never at module top level.
|
|
20
|
+
|
|
21
|
+
## 2. Scaffold the adapter
|
|
22
|
+
|
|
23
|
+
Create `src/viparse/extract/<format>.py` with an engine class/callable that:
|
|
24
|
+
|
|
25
|
+
- Declares which formats (magic-byte signatures) it handles.
|
|
26
|
+
- Lazy-imports its heavy dependency inside the method, raising a clear error that names the extra
|
|
27
|
+
to install if the import fails.
|
|
28
|
+
- Extracts text and attaches raw signals (e.g. run font names, embedded encoding hints) needed by
|
|
29
|
+
the normalizer — but performs **no** encoding conversion or NFC itself.
|
|
30
|
+
- Returns a `RawExtraction`.
|
|
31
|
+
|
|
32
|
+
## 3. Register it
|
|
33
|
+
|
|
34
|
+
Add the engine to the format→engine registry used by the `route` layer so magic-byte detection can
|
|
35
|
+
select it. Keep detection based on real bytes, not the filename extension.
|
|
36
|
+
|
|
37
|
+
## 4. Add a test
|
|
38
|
+
|
|
39
|
+
- Add a small, synthetic fixture under `tests/` (no sensitive or copyrighted documents).
|
|
40
|
+
- Test that the engine is selected for the format and returns the expected raw text and signals.
|
|
41
|
+
- If the dependency is heavy/optional, skip the test cleanly when the extra is not installed.
|
|
42
|
+
|
|
43
|
+
## 5. Verify
|
|
44
|
+
|
|
45
|
+
Run `scripts/dev.sh` (lint + type-check + test). Then open one PR titled `VIP-<id> Add <format> engine`.
|
|
46
|
+
|
|
47
|
+
## Checklist
|
|
48
|
+
|
|
49
|
+
- [ ] Wraps a maintained library — no hand-written parser
|
|
50
|
+
- [ ] Heavy dependency lazy-imported behind an extra
|
|
51
|
+
- [ ] No Vietnamese/encoding logic in the engine
|
|
52
|
+
- [ ] Registered for magic-byte routing
|
|
53
|
+
- [ ] Synthetic fixture + test, optional-dependency skip handled
|
|
54
|
+
- [ ] `scripts/dev.sh` passes
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: release
|
|
3
|
+
description: Cut a viparse release — bump the version, update the CHANGELOG, tag, and build the distribution. Use when publishing a new version of the package.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Cut a release
|
|
7
|
+
|
|
8
|
+
viparse uses a single source of truth for the version: `src/viparse/__init__.py` (`__version__`),
|
|
9
|
+
read by hatchling at build time. Releases are cut from a green `main`.
|
|
10
|
+
|
|
11
|
+
Follow these steps.
|
|
12
|
+
|
|
13
|
+
## 1. Pre-flight
|
|
14
|
+
|
|
15
|
+
- Ensure `main` is up to date and CI is green.
|
|
16
|
+
- Run `scripts/dev.sh` locally — lint, type-check, and tests must pass.
|
|
17
|
+
- Decide the new version per **SemVer** (MAJOR.MINOR.PATCH) based on what changed since the last tag.
|
|
18
|
+
|
|
19
|
+
## 2. Bump the version
|
|
20
|
+
|
|
21
|
+
- Edit `__version__` in `src/viparse/__init__.py` to the new version.
|
|
22
|
+
- This is the only place the version is defined — do not duplicate it in `pyproject.toml`
|
|
23
|
+
(it is `dynamic`).
|
|
24
|
+
|
|
25
|
+
## 3. Update the CHANGELOG
|
|
26
|
+
|
|
27
|
+
- Add a new `## <version> — <YYYY-MM-DD>` section at the top of `CHANGELOG.md`.
|
|
28
|
+
- Group entries under Added / Changed / Fixed / Removed. Summarize user-facing changes; link the
|
|
29
|
+
relevant `VIP-<id>` tasks. Keep it in English.
|
|
30
|
+
|
|
31
|
+
## 4. Land the bump
|
|
32
|
+
|
|
33
|
+
- Open one PR titled `VIP-<id> Release v<version>` with the version bump + CHANGELOG.
|
|
34
|
+
- Merge once CI is green.
|
|
35
|
+
|
|
36
|
+
## 5. Tag and build
|
|
37
|
+
|
|
38
|
+
From the merged `main`:
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
git tag -a v<version> -m "v<version>"
|
|
42
|
+
git push origin v<version>
|
|
43
|
+
scripts/dev.sh build # produces the wheel + sdist in dist/
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
- Verify the built artifacts carry the new version (`dist/viparse-<version>*`).
|
|
47
|
+
- Publish per the project's distribution process (e.g. PyPI) once configured.
|
|
48
|
+
|
|
49
|
+
## Checklist
|
|
50
|
+
|
|
51
|
+
- [ ] `main` green, `scripts/dev.sh` passes
|
|
52
|
+
- [ ] Version bumped only in `src/viparse/__init__.py`, per SemVer
|
|
53
|
+
- [ ] CHANGELOG updated with dated section linking `VIP-<id>`s
|
|
54
|
+
- [ ] Release PR merged on green CI
|
|
55
|
+
- [ ] Annotated tag `v<version>` pushed
|
|
56
|
+
- [ ] Wheel + sdist built and version-verified
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
version: 2
|
|
2
|
+
updates:
|
|
3
|
+
- package-ecosystem: pip
|
|
4
|
+
directory: "/"
|
|
5
|
+
schedule:
|
|
6
|
+
interval: weekly
|
|
7
|
+
groups:
|
|
8
|
+
python-dependencies:
|
|
9
|
+
patterns: ["*"]
|
|
10
|
+
|
|
11
|
+
- package-ecosystem: github-actions
|
|
12
|
+
directory: "/"
|
|
13
|
+
schedule:
|
|
14
|
+
interval: weekly
|
|
15
|
+
groups:
|
|
16
|
+
github-actions:
|
|
17
|
+
patterns: ["*"]
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
<!--
|
|
2
|
+
Keep PRs to one task. Title: `VIP-<id> <short imperative>`. No long description needed.
|
|
3
|
+
This checklist is the review contract; delete lines that genuinely do not apply.
|
|
4
|
+
-->
|
|
5
|
+
|
|
6
|
+
## Checklist
|
|
7
|
+
|
|
8
|
+
- [ ] One task, one commit — scope matches a single `VIP-<id>`
|
|
9
|
+
- [ ] Acceptance criteria from the spec are met
|
|
10
|
+
- [ ] Output stays Unicode **NFC** (if this touches text handling)
|
|
11
|
+
- [ ] No hand-written parser — engines wrapped behind thin adapters (if this touches extraction)
|
|
12
|
+
- [ ] Lint, type-check, and tests pass locally (`scripts/dev.sh`)
|
|
13
|
+
- [ ] All committed text is in English
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
concurrency:
|
|
9
|
+
group: ci-${{ github.ref }}
|
|
10
|
+
cancel-in-progress: true
|
|
11
|
+
|
|
12
|
+
jobs:
|
|
13
|
+
quality:
|
|
14
|
+
runs-on: ubuntu-latest
|
|
15
|
+
strategy:
|
|
16
|
+
fail-fast: false
|
|
17
|
+
matrix:
|
|
18
|
+
python-version: ["3.11", "3.12", "3.13"]
|
|
19
|
+
steps:
|
|
20
|
+
- uses: actions/checkout@v7
|
|
21
|
+
- uses: actions/setup-python@v6
|
|
22
|
+
with:
|
|
23
|
+
python-version: ${{ matrix.python-version }}
|
|
24
|
+
cache: pip
|
|
25
|
+
- name: Install
|
|
26
|
+
run: |
|
|
27
|
+
python -m pip install --upgrade pip
|
|
28
|
+
pip install -e ".[dev,office,pdf]"
|
|
29
|
+
- name: Lint (ruff)
|
|
30
|
+
run: bash scripts/dev.sh lint
|
|
31
|
+
- name: Type-check (mypy)
|
|
32
|
+
run: bash scripts/dev.sh type
|
|
33
|
+
- name: Test (pytest + coverage)
|
|
34
|
+
run: bash scripts/dev.sh test
|
|
35
|
+
|
|
36
|
+
audit:
|
|
37
|
+
runs-on: ubuntu-latest
|
|
38
|
+
steps:
|
|
39
|
+
- uses: actions/checkout@v7
|
|
40
|
+
- uses: actions/setup-python@v6
|
|
41
|
+
with:
|
|
42
|
+
python-version: "3.12"
|
|
43
|
+
cache: pip
|
|
44
|
+
- name: Install project
|
|
45
|
+
run: |
|
|
46
|
+
python -m pip install --upgrade pip
|
|
47
|
+
pip install -e .
|
|
48
|
+
- name: Audit dependencies (pip-audit)
|
|
49
|
+
run: |
|
|
50
|
+
pip install pip-audit
|
|
51
|
+
pip-audit --progress-spinner off
|
|
52
|
+
|
|
53
|
+
build:
|
|
54
|
+
runs-on: ubuntu-latest
|
|
55
|
+
steps:
|
|
56
|
+
- uses: actions/checkout@v7
|
|
57
|
+
- uses: actions/setup-python@v6
|
|
58
|
+
with:
|
|
59
|
+
python-version: "3.12"
|
|
60
|
+
cache: pip
|
|
61
|
+
- name: Build wheel and sdist
|
|
62
|
+
run: |
|
|
63
|
+
python -m pip install --upgrade pip build twine
|
|
64
|
+
python -m build
|
|
65
|
+
- name: Validate distribution metadata
|
|
66
|
+
run: twine check dist/*
|
|
67
|
+
|
|
68
|
+
# SBOM is informational and independent of the release artifact, so it lives in its own
|
|
69
|
+
# job (a transient tooling failure must never gate the build) and runs in a clean env
|
|
70
|
+
# holding only the package's runtime deps — not build/test tooling.
|
|
71
|
+
sbom:
|
|
72
|
+
runs-on: ubuntu-latest
|
|
73
|
+
steps:
|
|
74
|
+
- uses: actions/checkout@v7
|
|
75
|
+
- uses: actions/setup-python@v6
|
|
76
|
+
with:
|
|
77
|
+
python-version: "3.12"
|
|
78
|
+
cache: pip
|
|
79
|
+
- name: Generate SBOM (CycloneDX)
|
|
80
|
+
continue-on-error: true
|
|
81
|
+
run: |
|
|
82
|
+
python -m pip install --upgrade pip
|
|
83
|
+
pip install ".[all]" cyclonedx-bom
|
|
84
|
+
cyclonedx-py environment --pyproject pyproject.toml --of json -o sbom.json
|
|
85
|
+
- name: Upload SBOM
|
|
86
|
+
uses: actions/upload-artifact@v4
|
|
87
|
+
with:
|
|
88
|
+
name: sbom
|
|
89
|
+
path: sbom.json
|
|
90
|
+
if-no-files-found: warn
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
name: Publish
|
|
2
|
+
|
|
3
|
+
# Publish to PyPI via Trusted Publishing (OIDC) — no API token is stored.
|
|
4
|
+
# Fires when a GitHub Release is published; workflow_dispatch allows a manual
|
|
5
|
+
# run to backfill a release cut before this workflow existed (e.g. v0.1.0).
|
|
6
|
+
on:
|
|
7
|
+
release:
|
|
8
|
+
types: [published]
|
|
9
|
+
workflow_dispatch:
|
|
10
|
+
|
|
11
|
+
jobs:
|
|
12
|
+
publish:
|
|
13
|
+
runs-on: ubuntu-latest
|
|
14
|
+
environment:
|
|
15
|
+
name: pypi
|
|
16
|
+
url: https://pypi.org/p/viparse
|
|
17
|
+
permissions:
|
|
18
|
+
id-token: write # required for Trusted Publishing
|
|
19
|
+
steps:
|
|
20
|
+
- uses: actions/checkout@v7
|
|
21
|
+
- uses: actions/setup-python@v6
|
|
22
|
+
with:
|
|
23
|
+
python-version: "3.12"
|
|
24
|
+
cache: pip
|
|
25
|
+
- name: Build wheel and sdist
|
|
26
|
+
run: |
|
|
27
|
+
python -m pip install --upgrade pip build twine
|
|
28
|
+
python -m build
|
|
29
|
+
- name: Validate distribution metadata
|
|
30
|
+
run: twine check dist/*
|
|
31
|
+
- name: Publish to PyPI
|
|
32
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
viparse-0.1.0/.gitignore
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*.egg-info/
|
|
5
|
+
.eggs/
|
|
6
|
+
build/
|
|
7
|
+
dist/
|
|
8
|
+
.venv/
|
|
9
|
+
venv/
|
|
10
|
+
.env
|
|
11
|
+
|
|
12
|
+
# Tooling
|
|
13
|
+
.mypy_cache/
|
|
14
|
+
.ruff_cache/
|
|
15
|
+
.pytest_cache/
|
|
16
|
+
.coverage
|
|
17
|
+
htmlcov/
|
|
18
|
+
coverage.xml
|
|
19
|
+
|
|
20
|
+
# OS / editor
|
|
21
|
+
.DS_Store
|
|
22
|
+
.idea/
|
|
23
|
+
.vscode/
|
|
24
|
+
uv.lock
|
|
25
|
+
sbom.json
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# Pre-commit hooks for viparse.
|
|
2
|
+
# Install once with: pre-commit install
|
|
3
|
+
# Run on all files with: pre-commit run --all-files
|
|
4
|
+
minimum_pre_commit_version: "3.5.0"
|
|
5
|
+
|
|
6
|
+
repos:
|
|
7
|
+
- repo: https://github.com/astral-sh/ruff-pre-commit
|
|
8
|
+
rev: v0.6.9
|
|
9
|
+
hooks:
|
|
10
|
+
- id: ruff
|
|
11
|
+
args: [--fix]
|
|
12
|
+
- id: ruff-format
|
|
13
|
+
|
|
14
|
+
- repo: https://github.com/pre-commit/pre-commit-hooks
|
|
15
|
+
rev: v5.0.0
|
|
16
|
+
hooks:
|
|
17
|
+
- id: end-of-file-fixer
|
|
18
|
+
- id: trailing-whitespace
|
|
19
|
+
- id: check-yaml
|
|
20
|
+
- id: check-toml
|
|
21
|
+
- id: check-merge-conflict
|
|
22
|
+
- id: mixed-line-ending
|
|
23
|
+
args: [--fix=lf]
|
|
24
|
+
|
|
25
|
+
- repo: local
|
|
26
|
+
hooks:
|
|
27
|
+
- id: nfc-check
|
|
28
|
+
name: Unicode NFC check
|
|
29
|
+
description: Ensure text files are Unicode NFC-normalized (the viparse invariant).
|
|
30
|
+
entry: scripts/check_nfc.py
|
|
31
|
+
language: script
|
|
32
|
+
types: [text]
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to viparse are documented here. The format is based on
|
|
4
|
+
[Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the project adheres to
|
|
5
|
+
[Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
6
|
+
|
|
7
|
+
## [Unreleased]
|
|
8
|
+
|
|
9
|
+
## [0.1.0] — 2026-07-12
|
|
10
|
+
|
|
11
|
+
First tagged release. Covers the full M0–M5 feature set (VIP-1 … VIP-59).
|
|
12
|
+
|
|
13
|
+
### Added
|
|
14
|
+
|
|
15
|
+
- **Public API** — `viparse.load(source, *, output, encoding, ocr, normalize, max_bytes,
|
|
16
|
+
cache, chunk, settings)` and lazy `viparse.load_batch(...)`, returning Unicode-**NFC**
|
|
17
|
+
`Document`s as markdown / text / json.
|
|
18
|
+
- **Layered configuration** — `output` / `encoding` / `ocr` / `normalize` / `max_bytes` resolve
|
|
19
|
+
from function args → `VIPARSE_*` env vars → a `viparse.toml` file → the built-in defaults. A
|
|
20
|
+
validating `Settings` (via `load_settings()`) raises `ConfigError` on a bad value.
|
|
21
|
+
- **RAG chunking** — opt-in `chunk=ChunkOptions(max_tokens, overlap_tokens)` splits a document
|
|
22
|
+
into retrieval-sized, section-aware `Chunk`s (never splitting a table row) with per-chunk
|
|
23
|
+
`section` / `page` / `sheet` metadata and an ordinal `index`.
|
|
24
|
+
- **Framework integrations** — `to_langchain_documents(doc)` / `to_llamaindex_documents(doc)`
|
|
25
|
+
map a `Document` (chunk-aware) onto LangChain / LlamaIndex document types, provenance
|
|
26
|
+
flattened into their `metadata`. Lazy behind `viparse[langchain]` / `viparse[llamaindex]`.
|
|
27
|
+
- **CLI** — `viparse <files> -o md|text|json` (globs, directories, `--out`,
|
|
28
|
+
`--encoding`/`--ocr`/`--normalize`) and `viparse doctor` (engine + binary availability).
|
|
29
|
+
- **Extraction engines** — DOCX, XLSX, digital PDF, scanned PDF via OCR (`viparse[ocr]`),
|
|
30
|
+
and legacy binary `.doc`/`.xls` via LibreOffice — all thin adapters behind extras so the
|
|
31
|
+
`core` install stays dependency-free.
|
|
32
|
+
- **Vietnamese normalization (the moat)** — legacy **TCVN3 / VNI / VISCII** → Unicode NFC,
|
|
33
|
+
with font-signal detection, opt-in content-based detection (`encoding="auto"`), and text
|
|
34
|
+
cleanup. Output is always NFC.
|
|
35
|
+
- **Structured output** — headings, GFM tables, and a versioned JSON schema
|
|
36
|
+
(`viparse.SCHEMA_VERSION`).
|
|
37
|
+
- **Untrusted-input safety** — configurable file-size limit, zip-decompression-bomb guard,
|
|
38
|
+
and per-engine process timeouts (`UnsafeInput`).
|
|
39
|
+
- **Caching** — content-hash `MemoryCache` / `DiskCache` to skip re-parsing unchanged files.
|
|
40
|
+
- **Parallel batch** — `load_batch(..., workers=N)` with bounded concurrency and per-source
|
|
41
|
+
error isolation.
|
|
42
|
+
|
|
43
|
+
[Unreleased]: https://github.com/minhtridinh-kayden/viparse/compare/v0.1.0...HEAD
|
|
44
|
+
[0.1.0]: https://github.com/minhtridinh-kayden/viparse/releases/tag/v0.1.0
|
viparse-0.1.0/CLAUDE.md
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
# CLAUDE.md
|
|
2
|
+
|
|
3
|
+
Guidance for Claude Code when working in this repository.
|
|
4
|
+
|
|
5
|
+
## What viparse is
|
|
6
|
+
|
|
7
|
+
A Vietnamese-first document loader for RAG. It turns Vietnamese documents (including legacy
|
|
8
|
+
TCVN3/VNI/VISCII fonts, scanned PDFs, and old `.doc`/`.xls`) into clean Unicode **NFC**
|
|
9
|
+
Markdown/JSON, ready to push into a vector DB.
|
|
10
|
+
|
|
11
|
+
The differentiator (the moat) is the **Vietnamese normalization layer**: detect & convert legacy
|
|
12
|
+
encodings to Unicode and enforce NFC. Generic loaders parse the file but emit garbled diacritics
|
|
13
|
+
or wrong normalization; viparse fixes exactly that.
|
|
14
|
+
|
|
15
|
+
## Golden rules
|
|
16
|
+
|
|
17
|
+
These are non-negotiable. They override convenience.
|
|
18
|
+
|
|
19
|
+
1. **Never hand-write a parser.** Wrap a well-maintained engine behind a thin adapter. If an engine
|
|
20
|
+
gets a CVE or is abandoned, swap the adapter — don't reinvent it.
|
|
21
|
+
2. **Always NFC.** Output is Unicode NFC by default. Normalization is the product; never ship text
|
|
22
|
+
that isn't consistently normalized.
|
|
23
|
+
3. **Thin adapters.** Engines only extract and attach raw signals. No Vietnamese logic leaks into
|
|
24
|
+
an engine; that belongs in the normalization layer.
|
|
25
|
+
4. **Heavy engines are lazy.** OCR, LibreOffice, and other heavy dependencies are lazy-imported
|
|
26
|
+
behind extras (`viparse[ocr]`, `viparse[office]`). `core` must import with stdlib-only deps.
|
|
27
|
+
5. **Every task has acceptance criteria.** A task is done only when its criteria (from the spec)
|
|
28
|
+
are met and verified by a test.
|
|
29
|
+
|
|
30
|
+
## Architecture
|
|
31
|
+
|
|
32
|
+
The pipeline has four layers; each communicates through intermediate types so they stay decoupled
|
|
33
|
+
and independently testable:
|
|
34
|
+
|
|
35
|
+
```
|
|
36
|
+
viparse.load("file") -> list[Document]
|
|
37
|
+
|
|
38
|
+
route detect real format via magic bytes (not the extension); pick an engine
|
|
39
|
+
extract Engine adapter -> RawExtraction (raw text + font/encoding signals)
|
|
40
|
+
normalize Normalizer -> NormalizedDoc (legacy encoding -> Unicode, NFC, cleanup)
|
|
41
|
+
structure Renderer -> Document (text | markdown | json, + metadata)
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
- **Engines** (`extract`) are thin adapters wrapping well-maintained parse libraries. They carry
|
|
45
|
+
no Vietnamese logic — they only extract and attach raw signals (e.g. run font name).
|
|
46
|
+
- **Normalizers** (`normalize`) own the Vietnamese layer (the moat).
|
|
47
|
+
- **Renderers** (`structure`) produce the RAG-ready output and metadata.
|
|
48
|
+
|
|
49
|
+
Heavy dependencies (OCR, LibreOffice) are **lazy-imported** behind extras so `core` stays light.
|
|
50
|
+
|
|
51
|
+
## Repository layout
|
|
52
|
+
|
|
53
|
+
```
|
|
54
|
+
docs/specs/ Spec map: README + SPEC-0 .. SPEC-8 (source of truth for scope)
|
|
55
|
+
docs/ops/ Linear <-> GitHub workflow mapping
|
|
56
|
+
src/viparse/ Package source (added incrementally per spec)
|
|
57
|
+
tests/ Test suite + golden fixtures
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
Specs are the source of truth for scope. Before implementing, read the relevant
|
|
61
|
+
`docs/specs/SPEC-*.md`.
|
|
62
|
+
|
|
63
|
+
## Commands
|
|
64
|
+
|
|
65
|
+
> The Python toolchain is added in SPEC-0 (E0.6) and SPEC-5 (packaging). Until then, these are the
|
|
66
|
+
> intended entry points.
|
|
67
|
+
|
|
68
|
+
```bash
|
|
69
|
+
# Setup (once the package exists)
|
|
70
|
+
pip install -e ".[all]" # editable install with every extra
|
|
71
|
+
|
|
72
|
+
# Quality gates
|
|
73
|
+
ruff check . # lint
|
|
74
|
+
ruff format . # format
|
|
75
|
+
mypy src # type-check (strict)
|
|
76
|
+
pytest # tests
|
|
77
|
+
pytest --cov=viparse # tests with coverage
|
|
78
|
+
|
|
79
|
+
# CLI
|
|
80
|
+
viparse <file> -o md|text|json
|
|
81
|
+
viparse doctor # list available engines per installed extras
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
## Workflow
|
|
85
|
+
|
|
86
|
+
- Hierarchy: **SPEC -> Epic -> Task**. In Linear, an epic is an issue and a **task is a sub-issue**
|
|
87
|
+
(team key `VIP`). Sub-issues are created just-in-time per epic.
|
|
88
|
+
- **One task = one branch = one commit = one PR.** PR/commit title: `VIP-<id> <short imperative>`,
|
|
89
|
+
no description.
|
|
90
|
+
- All committed artifacts (docs, comments, identifiers, commit titles) are in **English**.
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# Contributing to viparse
|
|
2
|
+
|
|
3
|
+
viparse is built one task at a time. Every change traces back to a task in Linear (team key
|
|
4
|
+
`VIP`) and lands as a single, self-contained pull request.
|
|
5
|
+
|
|
6
|
+
## One task = one branch = one commit = one PR
|
|
7
|
+
|
|
8
|
+
- **Branch:** `vip-<id>-<short-slug>` — the Linear task id plus a short kebab-case slug.
|
|
9
|
+
Examples: `vip-31-add-minimal-pr-template`, `vip-33-add-scripts-dev-sh`.
|
|
10
|
+
(Linear also suggests this name on each task under "Copy git branch name".)
|
|
11
|
+
- **Commit / PR title:** `VIP-<id> <short imperative>` — e.g. `VIP-31 Add minimal PR template`.
|
|
12
|
+
No PR description is required; the checklist in the PR template is the review contract.
|
|
13
|
+
- Keep the PR scoped to its one task. If you discover unrelated work, open a new task for it.
|
|
14
|
+
|
|
15
|
+
## Before you push
|
|
16
|
+
|
|
17
|
+
Run the local quality gates — the same ones CI enforces:
|
|
18
|
+
|
|
19
|
+
```bash
|
|
20
|
+
scripts/dev.sh # lint (ruff) + type-check (mypy) + tests (pytest + coverage) + build
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
## House rules
|
|
24
|
+
|
|
25
|
+
- All committed artifacts — docs, comments, identifiers, commit and PR titles — are in **English**.
|
|
26
|
+
- Output text is Unicode **NFC**. Never ship text that is not consistently normalized.
|
|
27
|
+
- Never hand-write a parser: wrap a well-maintained engine behind a thin adapter.
|
|
28
|
+
- Read the relevant `docs/specs/SPEC-*.md` before implementing; the specs are the source of truth
|
|
29
|
+
for scope and acceptance criteria.
|