chunkr-rs 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- chunkr_rs-1.0.0/.github/workflows/publish.yml +140 -0
- chunkr_rs-1.0.0/.gitignore +9 -0
- chunkr_rs-1.0.0/CONTRIBUTION.md +35 -0
- chunkr_rs-1.0.0/Cargo.lock +865 -0
- chunkr_rs-1.0.0/Cargo.toml +31 -0
- chunkr_rs-1.0.0/LICENSE +21 -0
- chunkr_rs-1.0.0/PKG-INFO +190 -0
- chunkr_rs-1.0.0/PUBLISHING.md +68 -0
- chunkr_rs-1.0.0/README.md +159 -0
- chunkr_rs-1.0.0/chunkr.pyi +179 -0
- chunkr_rs-1.0.0/examples/advanced_strategies.rs +185 -0
- chunkr_rs-1.0.0/examples/chunk_by_chars.rs +22 -0
- chunkr_rs-1.0.0/examples/chunk_by_words.rs +22 -0
- chunkr_rs-1.0.0/examples/chunk_document.rs +37 -0
- chunkr_rs-1.0.0/examples/python_usage.py +126 -0
- chunkr_rs-1.0.0/pyproject.toml +53 -0
- chunkr_rs-1.0.0/src/chunker/agentic.rs +292 -0
- chunkr_rs-1.0.0/src/chunker/base.rs +77 -0
- chunkr_rs-1.0.0/src/chunker/char.rs +120 -0
- chunkr_rs-1.0.0/src/chunker/code.rs +217 -0
- chunkr_rs-1.0.0/src/chunker/contextual.rs +233 -0
- chunkr_rs-1.0.0/src/chunker/hierarchical.rs +221 -0
- chunkr_rs-1.0.0/src/chunker/html.rs +92 -0
- chunkr_rs-1.0.0/src/chunker/json.rs +200 -0
- chunkr_rs-1.0.0/src/chunker/markdown.rs +233 -0
- chunkr_rs-1.0.0/src/chunker/mod.rs +16 -0
- chunkr_rs-1.0.0/src/chunker/proposition.rs +296 -0
- chunkr_rs-1.0.0/src/chunker/query_aware.rs +204 -0
- chunkr_rs-1.0.0/src/chunker/recursive.rs +285 -0
- chunkr_rs-1.0.0/src/chunker/semantic.rs +306 -0
- chunkr_rs-1.0.0/src/chunker/sentence.rs +329 -0
- chunkr_rs-1.0.0/src/chunker/token.rs +188 -0
- chunkr_rs-1.0.0/src/chunker/word.rs +109 -0
- chunkr_rs-1.0.0/src/error.rs +49 -0
- chunkr_rs-1.0.0/src/lib.rs +46 -0
- chunkr_rs-1.0.0/src/loader/base.rs +4 -0
- chunkr_rs-1.0.0/src/loader/mod.rs +5 -0
- chunkr_rs-1.0.0/src/loader/pdf.rs +123 -0
- chunkr_rs-1.0.0/src/python.rs +617 -0
- chunkr_rs-1.0.0/src/structures/document.rs +53 -0
- chunkr_rs-1.0.0/src/structures/mod.rs +1 -0
- chunkr_rs-1.0.0/tests/chunker_tests.rs +508 -0
- chunkr_rs-1.0.0/tests/test_files/sample_doc.pdf +0 -0
- chunkr_rs-1.0.0/uv.lock +8 -0
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
name: CI & Release (PyPI & Crates.io)
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches:
|
|
6
|
+
- main
|
|
7
|
+
- master
|
|
8
|
+
tags:
|
|
9
|
+
- "v*"
|
|
10
|
+
pull_request:
|
|
11
|
+
workflow_dispatch:
|
|
12
|
+
|
|
13
|
+
permissions:
|
|
14
|
+
contents: read
|
|
15
|
+
id-token: write
|
|
16
|
+
|
|
17
|
+
jobs:
|
|
18
|
+
test-rust:
|
|
19
|
+
name: Rust Unit Tests
|
|
20
|
+
runs-on: ubuntu-latest
|
|
21
|
+
steps:
|
|
22
|
+
- uses: actions/checkout@v4
|
|
23
|
+
- uses: dtolnay/rust-toolchain@stable
|
|
24
|
+
- name: Run Cargo Tests
|
|
25
|
+
run: cargo test --release
|
|
26
|
+
|
|
27
|
+
linux:
|
|
28
|
+
runs-on: ubuntu-latest
|
|
29
|
+
strategy:
|
|
30
|
+
matrix:
|
|
31
|
+
target: [x86_64, aarch64, armv7, s390x, ppc64le]
|
|
32
|
+
steps:
|
|
33
|
+
- uses: actions/checkout@v4
|
|
34
|
+
- uses: actions/setup-python@v5
|
|
35
|
+
with:
|
|
36
|
+
python-version: '3.11'
|
|
37
|
+
- name: Build wheels
|
|
38
|
+
uses: PyO3/maturin-action@v1
|
|
39
|
+
with:
|
|
40
|
+
target: ${{ matrix.target }}
|
|
41
|
+
args: --release --out dist --find-interpreter
|
|
42
|
+
sccache: 'true'
|
|
43
|
+
manylinux: auto
|
|
44
|
+
- name: Upload wheels
|
|
45
|
+
uses: actions/upload-artifact@v4
|
|
46
|
+
with:
|
|
47
|
+
name: wheels-linux-${{ matrix.target }}
|
|
48
|
+
path: dist
|
|
49
|
+
|
|
50
|
+
windows:
|
|
51
|
+
runs-on: windows-latest
|
|
52
|
+
strategy:
|
|
53
|
+
matrix:
|
|
54
|
+
target: [x64, x86]
|
|
55
|
+
steps:
|
|
56
|
+
- uses: actions/checkout@v4
|
|
57
|
+
- uses: actions/setup-python@v5
|
|
58
|
+
with:
|
|
59
|
+
python-version: '3.11'
|
|
60
|
+
architecture: ${{ matrix.target }}
|
|
61
|
+
- name: Build wheels
|
|
62
|
+
uses: PyO3/maturin-action@v1
|
|
63
|
+
with:
|
|
64
|
+
target: ${{ matrix.target }}
|
|
65
|
+
args: --release --out dist --find-interpreter
|
|
66
|
+
sccache: 'true'
|
|
67
|
+
- name: Upload wheels
|
|
68
|
+
uses: actions/upload-artifact@v4
|
|
69
|
+
with:
|
|
70
|
+
name: wheels-windows-${{ matrix.target }}
|
|
71
|
+
path: dist
|
|
72
|
+
|
|
73
|
+
macos:
|
|
74
|
+
runs-on: macos-latest
|
|
75
|
+
strategy:
|
|
76
|
+
matrix:
|
|
77
|
+
target: [x86_64, aarch64]
|
|
78
|
+
steps:
|
|
79
|
+
- uses: actions/checkout@v4
|
|
80
|
+
- uses: actions/setup-python@v5
|
|
81
|
+
with:
|
|
82
|
+
python-version: '3.11'
|
|
83
|
+
- name: Build wheels
|
|
84
|
+
uses: PyO3/maturin-action@v1
|
|
85
|
+
with:
|
|
86
|
+
target: ${{ matrix.target }}
|
|
87
|
+
args: --release --out dist --find-interpreter
|
|
88
|
+
sccache: 'true'
|
|
89
|
+
- name: Upload wheels
|
|
90
|
+
uses: actions/upload-artifact@v4
|
|
91
|
+
with:
|
|
92
|
+
name: wheels-macos-${{ matrix.target }}
|
|
93
|
+
path: dist
|
|
94
|
+
|
|
95
|
+
sdist:
|
|
96
|
+
runs-on: ubuntu-latest
|
|
97
|
+
steps:
|
|
98
|
+
- uses: actions/checkout@v4
|
|
99
|
+
- name: Build sdist
|
|
100
|
+
uses: PyO3/maturin-action@v1
|
|
101
|
+
with:
|
|
102
|
+
command: sdist
|
|
103
|
+
args: --out dist
|
|
104
|
+
- name: Upload sdist
|
|
105
|
+
uses: actions/upload-artifact@v4
|
|
106
|
+
with:
|
|
107
|
+
name: wheels-sdist
|
|
108
|
+
path: dist
|
|
109
|
+
|
|
110
|
+
release-pypi:
|
|
111
|
+
name: Release to PyPI
|
|
112
|
+
runs-on: ubuntu-latest
|
|
113
|
+
if: startsWith(github.ref, 'refs/tags/v')
|
|
114
|
+
needs: [test-rust, linux, windows, macos, sdist]
|
|
115
|
+
permissions:
|
|
116
|
+
id-token: write
|
|
117
|
+
contents: read
|
|
118
|
+
steps:
|
|
119
|
+
- uses: actions/download-artifact@v4
|
|
120
|
+
with:
|
|
121
|
+
pattern: wheels-*
|
|
122
|
+
merge-multiple: true
|
|
123
|
+
path: dist
|
|
124
|
+
- name: Publish to PyPI
|
|
125
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
126
|
+
with:
|
|
127
|
+
packages-dir: dist/
|
|
128
|
+
password: ${{ secrets.PYPI_API_TOKEN }}
|
|
129
|
+
skip-existing: true
|
|
130
|
+
|
|
131
|
+
release-cargo:
|
|
132
|
+
name: Release to Crates.io
|
|
133
|
+
runs-on: ubuntu-latest
|
|
134
|
+
if: startsWith(github.ref, 'refs/tags/v')
|
|
135
|
+
needs: [test-rust]
|
|
136
|
+
steps:
|
|
137
|
+
- uses: actions/checkout@v4
|
|
138
|
+
- uses: dtolnay/rust-toolchain@stable
|
|
139
|
+
- name: Publish to Crates.io
|
|
140
|
+
run: cargo publish --token ${{ secrets.CARGO_REGISTRY_TOKEN }}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# CONTRIBUTION
|
|
2
|
+
|
|
3
|
+
We welcome your contributions to this project! Whether it's bug fixes, new features, or improvements to the documentation, we appreciate your help.
|
|
4
|
+
|
|
5
|
+
## Getting Started
|
|
6
|
+
Here's a quick guide to get you started:
|
|
7
|
+
|
|
8
|
+
**Fork the Repository**: Create your own copy of the repository on GitHub.
|
|
9
|
+
Clone the Fork: Download a copy of your forked repository to your local machine.
|
|
10
|
+
|
|
11
|
+
**Create a Branch**: Create a new branch for your changes.
|
|
12
|
+
|
|
13
|
+
**Make Changes**: Make your edits to the code.
|
|
14
|
+
|
|
15
|
+
**Test Changes**: Make sure your changes don't introduce any bugs. (This may involve running tests or manually trying out the functionality)
|
|
16
|
+
|
|
17
|
+
**Create a Pull Request**: Open a pull request to submit your changes for review.
|
|
18
|
+
|
|
19
|
+
## Reporting Bugs
|
|
20
|
+
If you find a bug in the project, please report it by opening a new issue on GitHub.
|
|
21
|
+
|
|
22
|
+
A good bug report should include:
|
|
23
|
+
|
|
24
|
+
- A short and descriptive title.
|
|
25
|
+
- A clear description of the bug.
|
|
26
|
+
- Steps to reproduce the bug (if possible).
|
|
27
|
+
- Expected behavior vs. actual behavior.
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
## License
|
|
31
|
+
Your contributions to this project will be licensed under the same license as the project itself (usually specified in a LICENSE file).
|
|
32
|
+
|
|
33
|
+
```
|
|
34
|
+
Thank you for you love and support.
|
|
35
|
+
```
|