tors 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tors-0.2.0/.github/ISSUE_TEMPLATE/bug_report.md +36 -0
- tors-0.2.0/.github/ISSUE_TEMPLATE/feature_request.md +21 -0
- tors-0.2.0/.github/dependabot.yml +36 -0
- tors-0.2.0/.github/pull_request_template.md +28 -0
- tors-0.2.0/.github/workflows/ci.yml +191 -0
- tors-0.2.0/.github/workflows/commitlint.yml +39 -0
- tors-0.2.0/.github/workflows/docs.yml +51 -0
- tors-0.2.0/.github/workflows/fuzz.yml +68 -0
- tors-0.2.0/.github/workflows/publish.yml +162 -0
- tors-0.2.0/.github/workflows/release-please.yml +67 -0
- tors-0.2.0/.github/workflows/wasm.yml +98 -0
- tors-0.2.0/.gitignore +32 -0
- tors-0.2.0/.release-please-manifest.json +3 -0
- tors-0.2.0/CHANGELOG.md +26 -0
- tors-0.2.0/CODE_OF_CONDUCT.md +127 -0
- tors-0.2.0/CONTRIBUTING.md +90 -0
- tors-0.2.0/Cargo.lock +1146 -0
- tors-0.2.0/Cargo.toml +147 -0
- tors-0.2.0/LICENSE +21 -0
- tors-0.2.0/Makefile +123 -0
- tors-0.2.0/PKG-INFO +452 -0
- tors-0.2.0/README.md +424 -0
- tors-0.2.0/SECURITY.md +37 -0
- tors-0.2.0/benches/bytes.rs +78 -0
- tors-0.2.0/benches/chunking.rs +79 -0
- tors-0.2.0/benches/common/mod.rs +44 -0
- tors-0.2.0/benches/diff.rs +352 -0
- tors-0.2.0/benches/encoding.rs +46 -0
- tors-0.2.0/benches/fence.rs +108 -0
- tors-0.2.0/benches/integrity.rs +104 -0
- tors-0.2.0/benches/normalize.rs +87 -0
- tors-0.2.0/benches/redaction.rs +116 -0
- tors-0.2.0/benches/retrieval.rs +177 -0
- tors-0.2.0/benches/search.rs +225 -0
- tors-0.2.0/benches/text.rs +265 -0
- tors-0.2.0/benches/utf8.rs +64 -0
- tors-0.2.0/deny.toml +57 -0
- tors-0.2.0/docs/api.md +2033 -0
- tors-0.2.0/docs/index.md +87 -0
- tors-0.2.0/docs/recipe-ingest.md +172 -0
- tors-0.2.0/docs/recipe-retrieval.md +171 -0
- tors-0.2.0/docs/superpowers/specs/2026-09-05-tors-v0.1-design.md +608 -0
- tors-0.2.0/mkdocs.yml +33 -0
- tors-0.2.0/pyproject.toml +97 -0
- tors-0.2.0/python/tors/__init__.py +141 -0
- tors-0.2.0/python/tors/__init__.pyi +723 -0
- tors-0.2.0/python/tors/aio.py +105 -0
- tors-0.2.0/python/tors/aio.pyi +93 -0
- tors-0.2.0/python/tors/py.typed +0 -0
- tors-0.2.0/release-please-config.json +15 -0
- tors-0.2.0/src/b64_impl.rs +511 -0
- tors-0.2.0/src/bm25_impl.rs +348 -0
- tors-0.2.0/src/chunk_by_segment_impl.rs +697 -0
- tors-0.2.0/src/chunk_hierarchical_impl.rs +390 -0
- tors-0.2.0/src/chunk_impl.rs +994 -0
- tors-0.2.0/src/decode_impl.rs +302 -0
- tors-0.2.0/src/diff_impl.rs +1693 -0
- tors-0.2.0/src/encoding_impl.rs +143 -0
- tors-0.2.0/src/fence_impl.rs +589 -0
- tors-0.2.0/src/finalize_impl.rs +271 -0
- tors-0.2.0/src/forms_impl.rs +276 -0
- tors-0.2.0/src/fuzzy_impl.rs +845 -0
- tors-0.2.0/src/grounded_impl.rs +249 -0
- tors-0.2.0/src/html_impl.rs +581 -0
- tors-0.2.0/src/html_table.rs +2320 -0
- tors-0.2.0/src/lib.rs +429 -0
- tors-0.2.0/src/merkle_impl.rs +237 -0
- tors-0.2.0/src/normalize_impl.rs +551 -0
- tors-0.2.0/src/phonetic_impl.rs +381 -0
- tors-0.2.0/src/pipeline_impl.rs +413 -0
- tors-0.2.0/src/py/_borrow.rs +159 -0
- tors-0.2.0/src/py/bm25.rs +99 -0
- tors-0.2.0/src/py/chunk.rs +457 -0
- tors-0.2.0/src/py/codec.rs +295 -0
- tors-0.2.0/src/py/compiled_patterns.rs +265 -0
- tors-0.2.0/src/py/diff.rs +133 -0
- tors-0.2.0/src/py/encoding.rs +23 -0
- tors-0.2.0/src/py/fence.rs +81 -0
- tors-0.2.0/src/py/forms.rs +46 -0
- tors-0.2.0/src/py/fuzzy.rs +146 -0
- tors-0.2.0/src/py/grounded.rs +47 -0
- tors-0.2.0/src/py/html.rs +99 -0
- tors-0.2.0/src/py/lemma_dict.rs +91 -0
- tors-0.2.0/src/py/merkle.rs +51 -0
- tors-0.2.0/src/py/mod.rs +72 -0
- tors-0.2.0/src/py/normalize.rs +58 -0
- tors-0.2.0/src/py/phonetic.rs +118 -0
- tors-0.2.0/src/py/pipeline.rs +123 -0
- tors-0.2.0/src/py/search.rs +300 -0
- tors-0.2.0/src/py/segmentation.rs +170 -0
- tors-0.2.0/src/py/simhash.rs +49 -0
- tors-0.2.0/src/py/tfidf.rs +92 -0
- tors-0.2.0/src/py/truncate.rs +54 -0
- tors-0.2.0/src/py/url.rs +58 -0
- tors-0.2.0/src/search_impl.rs +1808 -0
- tors-0.2.0/src/segmentation_impl.rs +485 -0
- tors-0.2.0/src/simhash_impl.rs +575 -0
- tors-0.2.0/src/tfidf_impl.rs +210 -0
- tors-0.2.0/src/tokenize_impl.rs +386 -0
- tors-0.2.0/src/truncate_impl.rs +342 -0
- tors-0.2.0/src/url_impl.rs +510 -0
- tors-0.2.0/src/utf16_impl.rs +404 -0
- tors-0.2.0/src/utf8_impl.rs +127 -0
- tors-0.2.0/tests/conftest.py +38 -0
- tors-0.2.0/tests/corpus/meeting_notes.md +24 -0
- tors-0.2.0/tests/corpus/notes.rtf +7 -0
- tors-0.2.0/tests/corpus/report.docx +0 -0
- tors-0.2.0/tests/corpus/report.pdf +0 -0
- tors-0.2.0/tests/corpus/samples.xlsx +0 -0
- tors-0.2.0/tests/documents.py +603 -0
- tors-0.2.0/tests/reference.py +481 -0
- tors-0.2.0/tests/test_aio.py +219 -0
- tors-0.2.0/tests/test_apply_pipeline.py +227 -0
- tors-0.2.0/tests/test_b64.py +121 -0
- tors-0.2.0/tests/test_b64_decode.py +459 -0
- tors-0.2.0/tests/test_bench_corpus_parity.py +414 -0
- tors-0.2.0/tests/test_bm25.py +289 -0
- tors-0.2.0/tests/test_chunk.py +132 -0
- tors-0.2.0/tests/test_chunk_hierarchical.py +257 -0
- tors-0.2.0/tests/test_chunk_text.py +507 -0
- tors-0.2.0/tests/test_compiled_patterns.py +644 -0
- tors-0.2.0/tests/test_decode_utf16.py +193 -0
- tors-0.2.0/tests/test_decode_utf8.py +372 -0
- tors-0.2.0/tests/test_diff_opcodes.py +654 -0
- tors-0.2.0/tests/test_diff_opcodes_lines.py +275 -0
- tors-0.2.0/tests/test_documents.py +324 -0
- tors-0.2.0/tests/test_encoding.py +165 -0
- tors-0.2.0/tests/test_fence.py +371 -0
- tors-0.2.0/tests/test_finalize.py +189 -0
- tors-0.2.0/tests/test_find_patterns.py +828 -0
- tors-0.2.0/tests/test_forms.py +245 -0
- tors-0.2.0/tests/test_fuzzy_match.py +374 -0
- tors-0.2.0/tests/test_gil_release.py +1697 -0
- tors-0.2.0/tests/test_grounded.py +212 -0
- tors-0.2.0/tests/test_html_unescape.py +492 -0
- tors-0.2.0/tests/test_merkle.py +283 -0
- tors-0.2.0/tests/test_normalize.py +290 -0
- tors-0.2.0/tests/test_parity.py +297 -0
- tors-0.2.0/tests/test_performance.py +297 -0
- tors-0.2.0/tests/test_phonetic.py +388 -0
- tors-0.2.0/tests/test_pyi_drift.py +261 -0
- tors-0.2.0/tests/test_replace_many.py +707 -0
- tors-0.2.0/tests/test_segmentation.py +433 -0
- tors-0.2.0/tests/test_sentence_bounds.py +328 -0
- tors-0.2.0/tests/test_simhash.py +213 -0
- tors-0.2.0/tests/test_simhash128.py +266 -0
- tors-0.2.0/tests/test_similarity.py +666 -0
- tors-0.2.0/tests/test_tfidf.py +253 -0
- tors-0.2.0/tests/test_truncate.py +139 -0
- tors-0.2.0/tests/test_url_quote.py +670 -0
- tors-0.2.0/tests/test_utf8_is_valid.py +203 -0
- tors-0.2.0/tools/bench_lemma_dict.py +71 -0
- tors-0.2.0/tools/gen_aio_stub.py +72 -0
- tors-0.2.0/tools/gen_html_table.py +121 -0
- tors-0.2.0/uv.lock +923 -0
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: Bug Report
|
|
3
|
+
about: Report a bug or unexpected behavior
|
|
4
|
+
labels: bug
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
## Description
|
|
8
|
+
|
|
9
|
+
A clear description of what the bug is.
|
|
10
|
+
|
|
11
|
+
## Reproduction
|
|
12
|
+
|
|
13
|
+
```python
|
|
14
|
+
import tors
|
|
15
|
+
|
|
16
|
+
# Minimal input that reproduces the issue
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
## Expected Behavior
|
|
20
|
+
|
|
21
|
+
What you expected `tors` to return.
|
|
22
|
+
|
|
23
|
+
## Actual Behavior
|
|
24
|
+
|
|
25
|
+
What it actually returned (include the full traceback/output).
|
|
26
|
+
|
|
27
|
+
## Environment
|
|
28
|
+
|
|
29
|
+
- tors version:
|
|
30
|
+
- Python version:
|
|
31
|
+
- OS / architecture:
|
|
32
|
+
- Installed from: (PyPI wheel / built from source)
|
|
33
|
+
|
|
34
|
+
## Additional Context
|
|
35
|
+
|
|
36
|
+
Any other context (logs, related issues, etc.).
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: Feature Request
|
|
3
|
+
about: Suggest a new feature or enhancement
|
|
4
|
+
labels: enhancement
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
## Problem
|
|
8
|
+
|
|
9
|
+
What problem does this feature solve? What's the current limitation?
|
|
10
|
+
|
|
11
|
+
## Proposed Solution
|
|
12
|
+
|
|
13
|
+
Describe the feature or change you'd like to see.
|
|
14
|
+
|
|
15
|
+
## Alternatives Considered
|
|
16
|
+
|
|
17
|
+
Any alternative approaches you've considered.
|
|
18
|
+
|
|
19
|
+
## Additional Context
|
|
20
|
+
|
|
21
|
+
Any other context, examples, or references.
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
version: 2
|
|
2
|
+
|
|
3
|
+
updates:
|
|
4
|
+
# Keeps the SHA pins in every workflow (ci.yml, publish.yml, docs.yml) current —
|
|
5
|
+
# SHA-pinning actions without this entry means never receiving upstream security fixes.
|
|
6
|
+
- package-ecosystem: github-actions
|
|
7
|
+
directory: /
|
|
8
|
+
schedule:
|
|
9
|
+
interval: weekly
|
|
10
|
+
commit-message:
|
|
11
|
+
prefix: "chore(deps-ci)"
|
|
12
|
+
groups:
|
|
13
|
+
github-actions:
|
|
14
|
+
patterns: ["*"]
|
|
15
|
+
|
|
16
|
+
- package-ecosystem: cargo
|
|
17
|
+
directory: /
|
|
18
|
+
schedule:
|
|
19
|
+
interval: weekly
|
|
20
|
+
commit-message:
|
|
21
|
+
prefix: "chore(deps-rust)"
|
|
22
|
+
groups:
|
|
23
|
+
rust-minor-patch:
|
|
24
|
+
update-types: ["minor", "patch"]
|
|
25
|
+
|
|
26
|
+
# `uv` is a first-class Dependabot ecosystem (understands pyproject.toml and uv.lock
|
|
27
|
+
# natively) — not `pip`, which reads pyproject.toml but doesn't maintain uv.lock.
|
|
28
|
+
- package-ecosystem: uv
|
|
29
|
+
directory: /
|
|
30
|
+
schedule:
|
|
31
|
+
interval: weekly
|
|
32
|
+
commit-message:
|
|
33
|
+
prefix: "chore(deps-py)"
|
|
34
|
+
groups:
|
|
35
|
+
python-minor-patch:
|
|
36
|
+
update-types: ["minor", "patch"]
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
## Summary
|
|
2
|
+
|
|
3
|
+
Brief description of what this PR does and why.
|
|
4
|
+
|
|
5
|
+
## Changes
|
|
6
|
+
|
|
7
|
+
- Change 1
|
|
8
|
+
- Change 2
|
|
9
|
+
|
|
10
|
+
## Testing
|
|
11
|
+
|
|
12
|
+
Describe how you tested these changes.
|
|
13
|
+
|
|
14
|
+
```bash
|
|
15
|
+
uv sync --locked
|
|
16
|
+
uv run --no-sync pytest -q
|
|
17
|
+
cargo test --no-default-features
|
|
18
|
+
cargo clippy --all-targets -- -D warnings
|
|
19
|
+
cargo fmt --check
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
## Docs
|
|
23
|
+
|
|
24
|
+
- [ ] `README.md` and `docs/` updated to match any behavior or API change in this PR.
|
|
25
|
+
|
|
26
|
+
## Related Issues
|
|
27
|
+
|
|
28
|
+
Fixes #123
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
# Manual trigger for debugging a workflow change without pushing a commit.
|
|
8
|
+
workflow_dispatch:
|
|
9
|
+
|
|
10
|
+
# Cancel the superseded run for the same branch when a new push lands:
|
|
11
|
+
# rapid follow-up commits stop burning runner minutes on an already-outdated run.
|
|
12
|
+
concurrency:
|
|
13
|
+
group: ${{ github.workflow }}-${{ github.ref }}
|
|
14
|
+
cancel-in-progress: true
|
|
15
|
+
|
|
16
|
+
# Least-privilege default: every job here reads and builds, none writes anything back to
|
|
17
|
+
# the repo or an external registry, so the top-level GITHUB_TOKEN needs no more than this.
|
|
18
|
+
permissions:
|
|
19
|
+
contents: read
|
|
20
|
+
|
|
21
|
+
env:
|
|
22
|
+
CARGO_TERM_COLOR: always
|
|
23
|
+
|
|
24
|
+
jobs:
|
|
25
|
+
lint:
|
|
26
|
+
# Runs once, not per matrix leg: Rust fmt/clippy is Python-version-agnostic, so the
|
|
27
|
+
# 3.10–3.14 matrix would buy five identical copies of the same signal.
|
|
28
|
+
runs-on: ubuntu-latest
|
|
29
|
+
steps:
|
|
30
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
31
|
+
|
|
32
|
+
- name: Install Rust
|
|
33
|
+
# The action installs with rustup's `--profile minimal`, which ships neither
|
|
34
|
+
# rustfmt nor clippy — this input is what makes the two steps below runnable.
|
|
35
|
+
uses: dtolnay/rust-toolchain@6bed0761d98439e5a578e2877258200ad565ba87 # stable
|
|
36
|
+
with:
|
|
37
|
+
components: rustfmt, clippy
|
|
38
|
+
|
|
39
|
+
- name: Cache cargo
|
|
40
|
+
# rust-cache, not sccache, in CI: every strategy pays its warm-up per job, and a
|
|
41
|
+
# ~10s clean build never amortizes sccache's install + cache-restore overhead on
|
|
42
|
+
# top of it. rust-cache already keys per job on the lockfile. sccache is wired
|
|
43
|
+
# where it does pay — the local edit/compile loop, via the Makefile (which does
|
|
44
|
+
# the same: sccache locally, no compiler cache in CI).
|
|
45
|
+
uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
|
|
46
|
+
|
|
47
|
+
- name: Cargo fmt
|
|
48
|
+
run: cargo fmt --check
|
|
49
|
+
|
|
50
|
+
- name: Cargo clippy
|
|
51
|
+
run: cargo clippy --all-targets -- -D warnings
|
|
52
|
+
|
|
53
|
+
- name: Cargo clippy (no default features)
|
|
54
|
+
# Second pass in the configuration `Cargo test` / `Compile benches` below actually
|
|
55
|
+
# build (--no-default-features): pyo3's cfg flags differ between the two configs,
|
|
56
|
+
# so each pass can surface lints in code the other never compiles.
|
|
57
|
+
run: cargo clippy --all-targets --no-default-features -- -D warnings
|
|
58
|
+
|
|
59
|
+
- name: Cargo deny (licenses, advisories, bans)
|
|
60
|
+
# The repo-wide, permanent licensing gate (spec v0.4 item 6): the
|
|
61
|
+
# permissive-only allowlist and the advisory/ban policy live in
|
|
62
|
+
# deny.toml; `make deny` runs the same three checks locally. The
|
|
63
|
+
# action ships a prebuilt cargo-deny binary, so the gate costs
|
|
64
|
+
# seconds, not the minutes a `cargo install cargo-deny --locked`
|
|
65
|
+
# would add to every run.
|
|
66
|
+
uses: EmbarkStudios/cargo-deny-action@3c6349835b2b7b196a839186cb8b78e02f7b5f25 # v2.1.1
|
|
67
|
+
with:
|
|
68
|
+
command-arguments: licenses advisories bans
|
|
69
|
+
|
|
70
|
+
- name: Install uv
|
|
71
|
+
# For the ruff step below: ruff is a dev-group dependency (pyproject.toml),
|
|
72
|
+
# so the lockfile — not this workflow — owns its version.
|
|
73
|
+
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
|
|
74
|
+
|
|
75
|
+
- name: Ruff
|
|
76
|
+
# The Python-side lint gate (config in pyproject.toml [tool.ruff];
|
|
77
|
+
# `make lint` runs the same check locally). --no-install-project keeps
|
|
78
|
+
# this a pure lint step: only the dev deps are installed, never the
|
|
79
|
+
# tors wheel (whose maturin build is the test job's business — the
|
|
80
|
+
# Rust gates above already compile the code it would wrap).
|
|
81
|
+
run: |
|
|
82
|
+
uv sync --locked --no-install-project
|
|
83
|
+
uv run --no-sync ruff check .
|
|
84
|
+
|
|
85
|
+
test:
|
|
86
|
+
# Cheap signals first: a fmt or clippy
|
|
87
|
+
# failure surfaces from `lint` in one fast job instead of after five matrix legs have
|
|
88
|
+
# each spent a uv sync + extension build on the same break.
|
|
89
|
+
needs: [lint]
|
|
90
|
+
runs-on: ubuntu-latest
|
|
91
|
+
strategy:
|
|
92
|
+
fail-fast: false
|
|
93
|
+
matrix:
|
|
94
|
+
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
|
|
95
|
+
steps:
|
|
96
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
97
|
+
|
|
98
|
+
- name: Install Rust
|
|
99
|
+
uses: dtolnay/rust-toolchain@6bed0761d98439e5a578e2877258200ad565ba87 # stable
|
|
100
|
+
|
|
101
|
+
- name: Cache cargo
|
|
102
|
+
# Same choice as the lint job's cache step (rationale there).
|
|
103
|
+
uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
|
|
104
|
+
|
|
105
|
+
- name: Cargo test
|
|
106
|
+
# extension-module doesn't link libpython, so plain `cargo test` needs it off.
|
|
107
|
+
# Rust-only; the crate isn't Python-version-specific, so this runs once per job
|
|
108
|
+
# rather than once per matrix entry.
|
|
109
|
+
if: matrix.python-version == '3.12'
|
|
110
|
+
run: cargo test --no-default-features
|
|
111
|
+
|
|
112
|
+
- name: Compile benches
|
|
113
|
+
# Compile-guard only: the bench docstring (benches/normalize.rs) and the design
|
|
114
|
+
# spec both promise that CI compiles the criterion benches; `--no-run` keeps that
|
|
115
|
+
# promise without spending runner time executing them. `--no-default-features`:
|
|
116
|
+
# same libpython-linking reason as `Cargo test` above — a bench binary links the rlib.
|
|
117
|
+
if: matrix.python-version == '3.12'
|
|
118
|
+
run: cargo bench --no-run --no-default-features
|
|
119
|
+
|
|
120
|
+
- name: Install uv
|
|
121
|
+
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
|
|
122
|
+
with:
|
|
123
|
+
python-version: ${{ matrix.python-version }}
|
|
124
|
+
|
|
125
|
+
- name: Sync dependencies and build the extension
|
|
126
|
+
# --locked: fail if uv.lock is out of date with pyproject.toml, rather than silently
|
|
127
|
+
# rewriting it.
|
|
128
|
+
run: uv sync --locked
|
|
129
|
+
|
|
130
|
+
- name: Pytest
|
|
131
|
+
# -m "not timing": the timing lane (tests/test_gil_release.py and
|
|
132
|
+
# tests/test_performance.py — the asyncio heartbeat-gap and wall-band
|
|
133
|
+
# measurement cells, marked `timing` in pyproject.toml) is slow and
|
|
134
|
+
# load-sensitive, so the matrix legs deselect it and ONE dedicated
|
|
135
|
+
# step on this 3.12 leg runs it right after: the GIL/perf contracts
|
|
136
|
+
# still gate every push, once, instead of being paid on all five
|
|
137
|
+
# legs. The marker split is exact (both lane files carry a module-level
|
|
138
|
+
# pytestmark, nothing else does), so no test loses its lane — verified
|
|
139
|
+
# by `pytest --collect-only -m timing` + `-m "not timing"` summing to
|
|
140
|
+
# the unfiltered count.
|
|
141
|
+
# -rP: also show the captured stdout of PASSING tests. The suite's
|
|
142
|
+
# print-only measurement cells (deliberately unasserted bands, e.g.
|
|
143
|
+
# b64_decode's wall parity) record their numbers via print, which the
|
|
144
|
+
# default -q report swallows — -rP surfaces them in the CI log for
|
|
145
|
+
# the record.
|
|
146
|
+
run: uv run --no-sync pytest -q -rP -m "not timing"
|
|
147
|
+
|
|
148
|
+
- name: Pytest (timing lane)
|
|
149
|
+
# The lane the matrix legs above deselect, run ONCE (the 3.12 leg,
|
|
150
|
+
# after the main pytest step): the GIL-release and wall-time
|
|
151
|
+
# measurement cells. -rP so the print-only recorded bands (and the
|
|
152
|
+
# b64 battery's recorded gh-145264 divergences, should this leg's
|
|
153
|
+
# stdlib predate the fix) surface in the log. Runs last on the leg so
|
|
154
|
+
# a timing failure never masks the fast lane's results.
|
|
155
|
+
if: matrix.python-version == '3.12'
|
|
156
|
+
run: uv run --no-sync pytest -q -rP -m timing
|
|
157
|
+
|
|
158
|
+
fuzz-smoke:
|
|
159
|
+
# A cheap regression net, not a real fuzzing campaign: 15s per target
|
|
160
|
+
# (CONTRIBUTING.md's `make fuzz-quick` runs the same targets for 30s
|
|
161
|
+
# locally) — enough to catch a reintroduced panic on every push without
|
|
162
|
+
# meaningfully lengthening CI. A real multi-hour campaign against one
|
|
163
|
+
# target of active suspicion is a local `cargo +nightly fuzz run
|
|
164
|
+
# <target>`, not something this job runs.
|
|
165
|
+
runs-on: ubuntu-latest
|
|
166
|
+
steps:
|
|
167
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
168
|
+
|
|
169
|
+
- name: Install Rust (nightly)
|
|
170
|
+
# cargo-fuzz's instrumented builds need nightly's -Z sanitizer flags;
|
|
171
|
+
# the rest of CI stays on stable (see the lint job's own note).
|
|
172
|
+
uses: dtolnay/rust-toolchain@6bed0761d98439e5a578e2877258200ad565ba87 # stable
|
|
173
|
+
with:
|
|
174
|
+
toolchain: nightly
|
|
175
|
+
|
|
176
|
+
- name: Cache cargo
|
|
177
|
+
uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
|
|
178
|
+
with:
|
|
179
|
+
workspaces: ". -> target\nfuzz -> target"
|
|
180
|
+
|
|
181
|
+
- name: Install cargo-fuzz
|
|
182
|
+
run: cargo install cargo-fuzz --locked
|
|
183
|
+
|
|
184
|
+
- name: Fuzz (15s per target)
|
|
185
|
+
working-directory: fuzz
|
|
186
|
+
run: |
|
|
187
|
+
set -e
|
|
188
|
+
for t in decode_utf8 decode_utf16 b64_decode html_unescape fence chunk_hierarchical normalize search segmentation diff phonetic bm25 tfidf; do
|
|
189
|
+
echo "=== $t ==="
|
|
190
|
+
cargo +nightly fuzz run "$t" -- -max_total_time=15 -close_fd_mask=3
|
|
191
|
+
done
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# Enforces Conventional Commits on the PR title, which is what release-please parses to
|
|
2
|
+
# decide the next version bump and CHANGELOG entry.
|
|
3
|
+
|
|
4
|
+
name: Commitlint
|
|
5
|
+
|
|
6
|
+
on:
|
|
7
|
+
pull_request:
|
|
8
|
+
types: [opened, edited, reopened, synchronize]
|
|
9
|
+
|
|
10
|
+
permissions:
|
|
11
|
+
contents: read
|
|
12
|
+
pull-requests: read
|
|
13
|
+
|
|
14
|
+
jobs:
|
|
15
|
+
commitlint:
|
|
16
|
+
runs-on: ubuntu-latest
|
|
17
|
+
timeout-minutes: 5
|
|
18
|
+
steps:
|
|
19
|
+
- uses: amannn/action-semantic-pull-request@48f256284bd46cdaab1048c3721360e808335d50 # v6.1.1
|
|
20
|
+
if: github.actor != 'dependabot[bot]' && github.actor != 'github-actions[bot]'
|
|
21
|
+
env:
|
|
22
|
+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
23
|
+
with:
|
|
24
|
+
types: |
|
|
25
|
+
feat
|
|
26
|
+
fix
|
|
27
|
+
docs
|
|
28
|
+
style
|
|
29
|
+
refactor
|
|
30
|
+
perf
|
|
31
|
+
test
|
|
32
|
+
chore
|
|
33
|
+
ci
|
|
34
|
+
build
|
|
35
|
+
subjectPattern: ^[a-z].+
|
|
36
|
+
subjectPatternError: |
|
|
37
|
+
The subject "{subject}" found in the pull request title "{title}"
|
|
38
|
+
didn't match the configured pattern. Please ensure that the subject
|
|
39
|
+
starts with a lowercase letter.
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# Builds the mkdocs site and deploys it to GitHub Pages via the modern OIDC-based deploy
|
|
2
|
+
# flow (actions/deploy-pages), not the legacy push-to-a-gh-pages-branch approach. Runs
|
|
3
|
+
# `mkdocs build --strict` so a broken nav entry or bad link fails CI instead of silently
|
|
4
|
+
# publishing a broken page.
|
|
5
|
+
|
|
6
|
+
name: docs
|
|
7
|
+
|
|
8
|
+
on:
|
|
9
|
+
push:
|
|
10
|
+
branches: [main]
|
|
11
|
+
paths:
|
|
12
|
+
- "docs/**"
|
|
13
|
+
- "mkdocs.yml"
|
|
14
|
+
- "pyproject.toml"
|
|
15
|
+
- "uv.lock"
|
|
16
|
+
- ".github/workflows/docs.yml"
|
|
17
|
+
workflow_dispatch: {}
|
|
18
|
+
|
|
19
|
+
permissions:
|
|
20
|
+
contents: read
|
|
21
|
+
|
|
22
|
+
jobs:
|
|
23
|
+
build:
|
|
24
|
+
runs-on: ubuntu-latest
|
|
25
|
+
steps:
|
|
26
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
27
|
+
- name: Install uv
|
|
28
|
+
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
|
|
29
|
+
- name: Sync docs dependencies
|
|
30
|
+
# --locked: fail if uv.lock is out of date, rather than silently rewriting it.
|
|
31
|
+
run: uv sync --locked --group docs --no-install-project --no-default-groups
|
|
32
|
+
- name: Build
|
|
33
|
+
run: uv run --no-sync mkdocs build --strict
|
|
34
|
+
- uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d # v6
|
|
35
|
+
- uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0
|
|
36
|
+
with:
|
|
37
|
+
path: site
|
|
38
|
+
|
|
39
|
+
deploy:
|
|
40
|
+
needs: build
|
|
41
|
+
runs-on: ubuntu-latest
|
|
42
|
+
if: github.ref == 'refs/heads/main'
|
|
43
|
+
permissions:
|
|
44
|
+
pages: write
|
|
45
|
+
id-token: write # Required by actions/deploy-pages to verify the deployment source.
|
|
46
|
+
environment:
|
|
47
|
+
name: github-pages
|
|
48
|
+
url: ${{ steps.deployment.outputs.page_url }}
|
|
49
|
+
steps:
|
|
50
|
+
- id: deployment
|
|
51
|
+
uses: actions/deploy-pages@368f82528645a54fb793d4d04e342629a3f51346 # v5
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
# The scheduled deep fuzzing pass. ci.yml's `fuzz-smoke` job runs the same
|
|
2
|
+
# targets for 15s on every push (a regression net); this job gives them a
|
|
3
|
+
# longer 30s budget on a weekly schedule, when runner minutes are cheap and
|
|
4
|
+
# nobody is waiting on the result, plus a manual `workflow_dispatch` trigger
|
|
5
|
+
# for running a pass on demand (e.g. after a change to a covered core).
|
|
6
|
+
# Crashes fail the job (no continue-on-error); the crash inputs libFuzzer
|
|
7
|
+
# writes to fuzz/artifacts/ and the accumulated corpus are uploaded on
|
|
8
|
+
# failure only, so a red run carries its repro case with it.
|
|
9
|
+
|
|
10
|
+
name: fuzz
|
|
11
|
+
|
|
12
|
+
on:
|
|
13
|
+
schedule:
|
|
14
|
+
# Mondays 03:20 UTC: off the top of the hour, away from the busiest
|
|
15
|
+
# minute of GitHub's scheduled-workload peak.
|
|
16
|
+
- cron: "20 3 * * 1"
|
|
17
|
+
workflow_dispatch:
|
|
18
|
+
|
|
19
|
+
permissions:
|
|
20
|
+
contents: read
|
|
21
|
+
|
|
22
|
+
env:
|
|
23
|
+
CARGO_TERM_COLOR: always
|
|
24
|
+
|
|
25
|
+
jobs:
|
|
26
|
+
fuzz:
|
|
27
|
+
runs-on: ubuntu-latest
|
|
28
|
+
steps:
|
|
29
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
30
|
+
|
|
31
|
+
- name: Install Rust (nightly)
|
|
32
|
+
# cargo-fuzz's instrumented builds need nightly's -Z sanitizer flags;
|
|
33
|
+
# the rest of CI stays on stable (see the lint job's own note in
|
|
34
|
+
# ci.yml).
|
|
35
|
+
uses: dtolnay/rust-toolchain@6bed0761d98439e5a578e2877258200ad565ba87 # stable
|
|
36
|
+
with:
|
|
37
|
+
toolchain: nightly
|
|
38
|
+
|
|
39
|
+
- name: Cache cargo
|
|
40
|
+
# Same choice as ci.yml's fuzz-smoke job (rationale there).
|
|
41
|
+
uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
|
|
42
|
+
with:
|
|
43
|
+
workspaces: ". -> target\nfuzz -> target"
|
|
44
|
+
|
|
45
|
+
- name: Install cargo-fuzz
|
|
46
|
+
run: cargo install cargo-fuzz --locked
|
|
47
|
+
|
|
48
|
+
- name: Fuzz (30s per target)
|
|
49
|
+
working-directory: fuzz
|
|
50
|
+
run: |
|
|
51
|
+
set -e
|
|
52
|
+
for t in decode_utf8 decode_utf16 b64_decode html_unescape fence chunk_hierarchical normalize search segmentation diff phonetic bm25 tfidf; do
|
|
53
|
+
echo "=== $t ==="
|
|
54
|
+
cargo +nightly fuzz run "$t" -- -max_total_time=30 -close_fd_mask=3
|
|
55
|
+
done
|
|
56
|
+
|
|
57
|
+
- name: Upload corpus and crash artifacts
|
|
58
|
+
# Failure only: a green run's corpus is reproducible from the seed
|
|
59
|
+
# inputs, but a red run's crash file is the repro case the report
|
|
60
|
+
# needs (minimize it locally with `cargo fuzz tmin`).
|
|
61
|
+
if: failure()
|
|
62
|
+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
63
|
+
with:
|
|
64
|
+
name: fuzz-crash-artifacts
|
|
65
|
+
path: |
|
|
66
|
+
fuzz/artifacts
|
|
67
|
+
fuzz/corpus
|
|
68
|
+
if-no-files-found: ignore
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
# Builds abi3 wheels (one per platform, valid for every Python 3.10+ per Cargo.toml's
|
|
2
|
+
# `abi3-py310` feature — no per-Python-version build matrix needed) plus an sdist, then
|
|
3
|
+
# publishes to PyPI via Trusted Publishing (OIDC) on a release-please tag. No PyPI API
|
|
4
|
+
# token exists anywhere in this repo — `id-token: write` is the whole auth story.
|
|
5
|
+
#
|
|
6
|
+
# Platform set is the mainstream cross-platform tier most serious PyO3 packages ship
|
|
7
|
+
# (Linux manylinux + musllinux on x86_64/aarch64, macOS x86_64/aarch64, Windows x64/arm64)
|
|
8
|
+
# — not maturin generate-ci's full exotic-target sprawl (32-bit x86, armv7, s390x, ppc64le,
|
|
9
|
+
# emscripten, android), which stays out until a real user asks for one of those.
|
|
10
|
+
#
|
|
11
|
+
# Tag-trigger caveat: release-please publishes its tag using GITHUB_TOKEN by default, and
|
|
12
|
+
# GitHub does not start workflow runs from GITHUB_TOKEN-authored events — so this workflow's
|
|
13
|
+
# `push: tags` trigger will not fire automatically unless release-please.yml is given a
|
|
14
|
+
# GitHub App token (see that workflow's header). Until one is configured, run this via
|
|
15
|
+
# `workflow_dispatch` against the new tag after a release PR merges.
|
|
16
|
+
#
|
|
17
|
+
# Job order is deliberately cheapest-first: wheels-linux (and sdist) run on ubuntu-22.04,
|
|
18
|
+
# by far the cheapest and fastest GitHub-hosted runner. wheels-other (macOS, Windows —
|
|
19
|
+
# both several times the per-minute cost of Linux, and macOS/Windows arm64 runners are
|
|
20
|
+
# scarcer capacity) only starts once every Linux leg has passed, via `needs`. A correctness
|
|
21
|
+
# bug that fails on Linux therefore never burns the expensive runners at all.
|
|
22
|
+
|
|
23
|
+
name: publish
|
|
24
|
+
|
|
25
|
+
on:
|
|
26
|
+
push:
|
|
27
|
+
tags: ["v[0-9]+.[0-9]+.[0-9]+"]
|
|
28
|
+
workflow_dispatch: {}
|
|
29
|
+
|
|
30
|
+
permissions:
|
|
31
|
+
contents: read
|
|
32
|
+
|
|
33
|
+
jobs:
|
|
34
|
+
wheels-linux:
|
|
35
|
+
strategy:
|
|
36
|
+
fail-fast: false
|
|
37
|
+
matrix:
|
|
38
|
+
include:
|
|
39
|
+
- target: x86_64
|
|
40
|
+
manylinux: auto
|
|
41
|
+
- target: aarch64
|
|
42
|
+
manylinux: auto
|
|
43
|
+
- target: x86_64
|
|
44
|
+
manylinux: musllinux_1_2
|
|
45
|
+
- target: aarch64
|
|
46
|
+
manylinux: musllinux_1_2
|
|
47
|
+
runs-on: ubuntu-22.04
|
|
48
|
+
steps:
|
|
49
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
50
|
+
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
|
51
|
+
with:
|
|
52
|
+
python-version: "3.13"
|
|
53
|
+
- name: Build wheel
|
|
54
|
+
uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0
|
|
55
|
+
with:
|
|
56
|
+
target: ${{ matrix.target }}
|
|
57
|
+
args: --release --out dist
|
|
58
|
+
manylinux: ${{ matrix.manylinux }}
|
|
59
|
+
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
60
|
+
with:
|
|
61
|
+
name: wheels-linux-${{ matrix.target }}-${{ matrix.manylinux }}
|
|
62
|
+
path: dist
|
|
63
|
+
|
|
64
|
+
sdist:
|
|
65
|
+
runs-on: ubuntu-latest
|
|
66
|
+
steps:
|
|
67
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
68
|
+
- name: Build sdist
|
|
69
|
+
uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0
|
|
70
|
+
with:
|
|
71
|
+
command: sdist
|
|
72
|
+
args: --out dist
|
|
73
|
+
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
74
|
+
with:
|
|
75
|
+
name: sdist
|
|
76
|
+
path: dist
|
|
77
|
+
|
|
78
|
+
wheels-other:
|
|
79
|
+
needs: wheels-linux
|
|
80
|
+
strategy:
|
|
81
|
+
fail-fast: false
|
|
82
|
+
matrix:
|
|
83
|
+
include:
|
|
84
|
+
- os: macos-15-intel
|
|
85
|
+
target: x86_64
|
|
86
|
+
- os: macos-latest
|
|
87
|
+
target: aarch64
|
|
88
|
+
- os: windows-latest
|
|
89
|
+
target: x64
|
|
90
|
+
python_arch: x64
|
|
91
|
+
- os: windows-11-arm
|
|
92
|
+
target: aarch64
|
|
93
|
+
python_arch: arm64
|
|
94
|
+
runs-on: ${{ matrix.os }}
|
|
95
|
+
steps:
|
|
96
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
97
|
+
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
|
98
|
+
with:
|
|
99
|
+
python-version: "3.13"
|
|
100
|
+
architecture: ${{ matrix.python_arch || 'x64' }}
|
|
101
|
+
- name: Build wheel
|
|
102
|
+
uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0
|
|
103
|
+
with:
|
|
104
|
+
target: ${{ matrix.target }}
|
|
105
|
+
args: --release --out dist
|
|
106
|
+
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
107
|
+
with:
|
|
108
|
+
name: wheels-${{ matrix.os }}-${{ matrix.target }}
|
|
109
|
+
path: dist
|
|
110
|
+
|
|
111
|
+
publish:
|
|
112
|
+
name: Publish to PyPI
|
|
113
|
+
needs: [wheels-linux, wheels-other, sdist]
|
|
114
|
+
runs-on: ubuntu-latest
|
|
115
|
+
environment: pypi
|
|
116
|
+
permissions:
|
|
117
|
+
id-token: write # Trusted Publishing — no API token secret needed or used.
|
|
118
|
+
steps:
|
|
119
|
+
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
|
120
|
+
with:
|
|
121
|
+
pattern: "wheels-*"
|
|
122
|
+
path: dist
|
|
123
|
+
merge-multiple: true
|
|
124
|
+
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
|
125
|
+
with:
|
|
126
|
+
name: sdist
|
|
127
|
+
path: dist
|
|
128
|
+
- name: Install uv
|
|
129
|
+
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
|
|
130
|
+
- name: Publish
|
|
131
|
+
run: uv publish --trusted-publishing automatic dist/*
|
|
132
|
+
|
|
133
|
+
publish-crate:
|
|
134
|
+
name: Publish to crates.io
|
|
135
|
+
# Independent of the PyPI leg above: the crate (published as `tors-core`, see
|
|
136
|
+
# Cargo.toml's own comment on the name collision) needs no wheel artifacts, just the
|
|
137
|
+
# source tree and Cargo.lock, so it runs in parallel rather than behind `needs`.
|
|
138
|
+
#
|
|
139
|
+
# First release only: crates.io has no PyPI-style "pending trusted publisher" for a
|
|
140
|
+
# crate that has never been published, so the very first `tors-core` release must be
|
|
141
|
+
# `cargo publish`d manually (from a machine already `cargo login`-ed). Only after that
|
|
142
|
+
# succeeds can a Trusted Publisher be configured in the crate's crates.io settings
|
|
143
|
+
# (GitHub owner `AZX-PBC-OSS`, repo `tors`, workflow `publish.yml`) — this job is that
|
|
144
|
+
# configuration's target and does nothing until it exists.
|
|
145
|
+
runs-on: ubuntu-latest
|
|
146
|
+
environment: crates-io
|
|
147
|
+
permissions:
|
|
148
|
+
id-token: write # Trusted Publishing (OIDC) — no API token secret needed or used.
|
|
149
|
+
steps:
|
|
150
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
151
|
+
- name: Install Rust
|
|
152
|
+
uses: dtolnay/rust-toolchain@6bed0761d98439e5a578e2877258200ad565ba87 # stable
|
|
153
|
+
- name: Authenticate with crates.io
|
|
154
|
+
id: auth
|
|
155
|
+
uses: rust-lang/crates-io-auth-action@c6f97d42243bad5fab37ca0427f495c86d5b1a18 # v1.0.5
|
|
156
|
+
- name: Publish
|
|
157
|
+
# --no-default-features: `extension-module` (the crate's default) doesn't link
|
|
158
|
+
# libpython, which breaks a plain `cargo publish` verification build the same way
|
|
159
|
+
# it breaks `cargo test`/`cargo build` — see Cargo.toml's own feature comment.
|
|
160
|
+
env:
|
|
161
|
+
CARGO_REGISTRY_TOKEN: ${{ steps.auth.outputs.token }}
|
|
162
|
+
run: cargo publish -p tors-core --no-default-features
|