tors 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (155) hide show
  1. tors-0.2.0/.github/ISSUE_TEMPLATE/bug_report.md +36 -0
  2. tors-0.2.0/.github/ISSUE_TEMPLATE/feature_request.md +21 -0
  3. tors-0.2.0/.github/dependabot.yml +36 -0
  4. tors-0.2.0/.github/pull_request_template.md +28 -0
  5. tors-0.2.0/.github/workflows/ci.yml +191 -0
  6. tors-0.2.0/.github/workflows/commitlint.yml +39 -0
  7. tors-0.2.0/.github/workflows/docs.yml +51 -0
  8. tors-0.2.0/.github/workflows/fuzz.yml +68 -0
  9. tors-0.2.0/.github/workflows/publish.yml +162 -0
  10. tors-0.2.0/.github/workflows/release-please.yml +67 -0
  11. tors-0.2.0/.github/workflows/wasm.yml +98 -0
  12. tors-0.2.0/.gitignore +32 -0
  13. tors-0.2.0/.release-please-manifest.json +3 -0
  14. tors-0.2.0/CHANGELOG.md +26 -0
  15. tors-0.2.0/CODE_OF_CONDUCT.md +127 -0
  16. tors-0.2.0/CONTRIBUTING.md +90 -0
  17. tors-0.2.0/Cargo.lock +1146 -0
  18. tors-0.2.0/Cargo.toml +147 -0
  19. tors-0.2.0/LICENSE +21 -0
  20. tors-0.2.0/Makefile +123 -0
  21. tors-0.2.0/PKG-INFO +452 -0
  22. tors-0.2.0/README.md +424 -0
  23. tors-0.2.0/SECURITY.md +37 -0
  24. tors-0.2.0/benches/bytes.rs +78 -0
  25. tors-0.2.0/benches/chunking.rs +79 -0
  26. tors-0.2.0/benches/common/mod.rs +44 -0
  27. tors-0.2.0/benches/diff.rs +352 -0
  28. tors-0.2.0/benches/encoding.rs +46 -0
  29. tors-0.2.0/benches/fence.rs +108 -0
  30. tors-0.2.0/benches/integrity.rs +104 -0
  31. tors-0.2.0/benches/normalize.rs +87 -0
  32. tors-0.2.0/benches/redaction.rs +116 -0
  33. tors-0.2.0/benches/retrieval.rs +177 -0
  34. tors-0.2.0/benches/search.rs +225 -0
  35. tors-0.2.0/benches/text.rs +265 -0
  36. tors-0.2.0/benches/utf8.rs +64 -0
  37. tors-0.2.0/deny.toml +57 -0
  38. tors-0.2.0/docs/api.md +2033 -0
  39. tors-0.2.0/docs/index.md +87 -0
  40. tors-0.2.0/docs/recipe-ingest.md +172 -0
  41. tors-0.2.0/docs/recipe-retrieval.md +171 -0
  42. tors-0.2.0/docs/superpowers/specs/2026-09-05-tors-v0.1-design.md +608 -0
  43. tors-0.2.0/mkdocs.yml +33 -0
  44. tors-0.2.0/pyproject.toml +97 -0
  45. tors-0.2.0/python/tors/__init__.py +141 -0
  46. tors-0.2.0/python/tors/__init__.pyi +723 -0
  47. tors-0.2.0/python/tors/aio.py +105 -0
  48. tors-0.2.0/python/tors/aio.pyi +93 -0
  49. tors-0.2.0/python/tors/py.typed +0 -0
  50. tors-0.2.0/release-please-config.json +15 -0
  51. tors-0.2.0/src/b64_impl.rs +511 -0
  52. tors-0.2.0/src/bm25_impl.rs +348 -0
  53. tors-0.2.0/src/chunk_by_segment_impl.rs +697 -0
  54. tors-0.2.0/src/chunk_hierarchical_impl.rs +390 -0
  55. tors-0.2.0/src/chunk_impl.rs +994 -0
  56. tors-0.2.0/src/decode_impl.rs +302 -0
  57. tors-0.2.0/src/diff_impl.rs +1693 -0
  58. tors-0.2.0/src/encoding_impl.rs +143 -0
  59. tors-0.2.0/src/fence_impl.rs +589 -0
  60. tors-0.2.0/src/finalize_impl.rs +271 -0
  61. tors-0.2.0/src/forms_impl.rs +276 -0
  62. tors-0.2.0/src/fuzzy_impl.rs +845 -0
  63. tors-0.2.0/src/grounded_impl.rs +249 -0
  64. tors-0.2.0/src/html_impl.rs +581 -0
  65. tors-0.2.0/src/html_table.rs +2320 -0
  66. tors-0.2.0/src/lib.rs +429 -0
  67. tors-0.2.0/src/merkle_impl.rs +237 -0
  68. tors-0.2.0/src/normalize_impl.rs +551 -0
  69. tors-0.2.0/src/phonetic_impl.rs +381 -0
  70. tors-0.2.0/src/pipeline_impl.rs +413 -0
  71. tors-0.2.0/src/py/_borrow.rs +159 -0
  72. tors-0.2.0/src/py/bm25.rs +99 -0
  73. tors-0.2.0/src/py/chunk.rs +457 -0
  74. tors-0.2.0/src/py/codec.rs +295 -0
  75. tors-0.2.0/src/py/compiled_patterns.rs +265 -0
  76. tors-0.2.0/src/py/diff.rs +133 -0
  77. tors-0.2.0/src/py/encoding.rs +23 -0
  78. tors-0.2.0/src/py/fence.rs +81 -0
  79. tors-0.2.0/src/py/forms.rs +46 -0
  80. tors-0.2.0/src/py/fuzzy.rs +146 -0
  81. tors-0.2.0/src/py/grounded.rs +47 -0
  82. tors-0.2.0/src/py/html.rs +99 -0
  83. tors-0.2.0/src/py/lemma_dict.rs +91 -0
  84. tors-0.2.0/src/py/merkle.rs +51 -0
  85. tors-0.2.0/src/py/mod.rs +72 -0
  86. tors-0.2.0/src/py/normalize.rs +58 -0
  87. tors-0.2.0/src/py/phonetic.rs +118 -0
  88. tors-0.2.0/src/py/pipeline.rs +123 -0
  89. tors-0.2.0/src/py/search.rs +300 -0
  90. tors-0.2.0/src/py/segmentation.rs +170 -0
  91. tors-0.2.0/src/py/simhash.rs +49 -0
  92. tors-0.2.0/src/py/tfidf.rs +92 -0
  93. tors-0.2.0/src/py/truncate.rs +54 -0
  94. tors-0.2.0/src/py/url.rs +58 -0
  95. tors-0.2.0/src/search_impl.rs +1808 -0
  96. tors-0.2.0/src/segmentation_impl.rs +485 -0
  97. tors-0.2.0/src/simhash_impl.rs +575 -0
  98. tors-0.2.0/src/tfidf_impl.rs +210 -0
  99. tors-0.2.0/src/tokenize_impl.rs +386 -0
  100. tors-0.2.0/src/truncate_impl.rs +342 -0
  101. tors-0.2.0/src/url_impl.rs +510 -0
  102. tors-0.2.0/src/utf16_impl.rs +404 -0
  103. tors-0.2.0/src/utf8_impl.rs +127 -0
  104. tors-0.2.0/tests/conftest.py +38 -0
  105. tors-0.2.0/tests/corpus/meeting_notes.md +24 -0
  106. tors-0.2.0/tests/corpus/notes.rtf +7 -0
  107. tors-0.2.0/tests/corpus/report.docx +0 -0
  108. tors-0.2.0/tests/corpus/report.pdf +0 -0
  109. tors-0.2.0/tests/corpus/samples.xlsx +0 -0
  110. tors-0.2.0/tests/documents.py +603 -0
  111. tors-0.2.0/tests/reference.py +481 -0
  112. tors-0.2.0/tests/test_aio.py +219 -0
  113. tors-0.2.0/tests/test_apply_pipeline.py +227 -0
  114. tors-0.2.0/tests/test_b64.py +121 -0
  115. tors-0.2.0/tests/test_b64_decode.py +459 -0
  116. tors-0.2.0/tests/test_bench_corpus_parity.py +414 -0
  117. tors-0.2.0/tests/test_bm25.py +289 -0
  118. tors-0.2.0/tests/test_chunk.py +132 -0
  119. tors-0.2.0/tests/test_chunk_hierarchical.py +257 -0
  120. tors-0.2.0/tests/test_chunk_text.py +507 -0
  121. tors-0.2.0/tests/test_compiled_patterns.py +644 -0
  122. tors-0.2.0/tests/test_decode_utf16.py +193 -0
  123. tors-0.2.0/tests/test_decode_utf8.py +372 -0
  124. tors-0.2.0/tests/test_diff_opcodes.py +654 -0
  125. tors-0.2.0/tests/test_diff_opcodes_lines.py +275 -0
  126. tors-0.2.0/tests/test_documents.py +324 -0
  127. tors-0.2.0/tests/test_encoding.py +165 -0
  128. tors-0.2.0/tests/test_fence.py +371 -0
  129. tors-0.2.0/tests/test_finalize.py +189 -0
  130. tors-0.2.0/tests/test_find_patterns.py +828 -0
  131. tors-0.2.0/tests/test_forms.py +245 -0
  132. tors-0.2.0/tests/test_fuzzy_match.py +374 -0
  133. tors-0.2.0/tests/test_gil_release.py +1697 -0
  134. tors-0.2.0/tests/test_grounded.py +212 -0
  135. tors-0.2.0/tests/test_html_unescape.py +492 -0
  136. tors-0.2.0/tests/test_merkle.py +283 -0
  137. tors-0.2.0/tests/test_normalize.py +290 -0
  138. tors-0.2.0/tests/test_parity.py +297 -0
  139. tors-0.2.0/tests/test_performance.py +297 -0
  140. tors-0.2.0/tests/test_phonetic.py +388 -0
  141. tors-0.2.0/tests/test_pyi_drift.py +261 -0
  142. tors-0.2.0/tests/test_replace_many.py +707 -0
  143. tors-0.2.0/tests/test_segmentation.py +433 -0
  144. tors-0.2.0/tests/test_sentence_bounds.py +328 -0
  145. tors-0.2.0/tests/test_simhash.py +213 -0
  146. tors-0.2.0/tests/test_simhash128.py +266 -0
  147. tors-0.2.0/tests/test_similarity.py +666 -0
  148. tors-0.2.0/tests/test_tfidf.py +253 -0
  149. tors-0.2.0/tests/test_truncate.py +139 -0
  150. tors-0.2.0/tests/test_url_quote.py +670 -0
  151. tors-0.2.0/tests/test_utf8_is_valid.py +203 -0
  152. tors-0.2.0/tools/bench_lemma_dict.py +71 -0
  153. tors-0.2.0/tools/gen_aio_stub.py +72 -0
  154. tors-0.2.0/tools/gen_html_table.py +121 -0
  155. tors-0.2.0/uv.lock +923 -0
@@ -0,0 +1,36 @@
1
+ ---
2
+ name: Bug Report
3
+ about: Report a bug or unexpected behavior
4
+ labels: bug
5
+ ---
6
+
7
+ ## Description
8
+
9
+ A clear description of what the bug is.
10
+
11
+ ## Reproduction
12
+
13
+ ```python
14
+ import tors
15
+
16
+ # Minimal input that reproduces the issue
17
+ ```
18
+
19
+ ## Expected Behavior
20
+
21
+ What you expected `tors` to return.
22
+
23
+ ## Actual Behavior
24
+
25
+ What it actually returned (include the full traceback/output).
26
+
27
+ ## Environment
28
+
29
+ - tors version:
30
+ - Python version:
31
+ - OS / architecture:
32
+ - Installed from: (PyPI wheel / built from source)
33
+
34
+ ## Additional Context
35
+
36
+ Any other context (logs, related issues, etc.).
@@ -0,0 +1,21 @@
1
+ ---
2
+ name: Feature Request
3
+ about: Suggest a new feature or enhancement
4
+ labels: enhancement
5
+ ---
6
+
7
+ ## Problem
8
+
9
+ What problem does this feature solve? What's the current limitation?
10
+
11
+ ## Proposed Solution
12
+
13
+ Describe the feature or change you'd like to see.
14
+
15
+ ## Alternatives Considered
16
+
17
+ Any alternative approaches you've considered.
18
+
19
+ ## Additional Context
20
+
21
+ Any other context, examples, or references.
@@ -0,0 +1,36 @@
1
+ version: 2
2
+
3
+ updates:
4
+ # Keeps the SHA pins in every workflow (ci.yml, publish.yml, docs.yml) current —
5
+ # SHA-pinning actions without this entry means never receiving upstream security fixes.
6
+ - package-ecosystem: github-actions
7
+ directory: /
8
+ schedule:
9
+ interval: weekly
10
+ commit-message:
11
+ prefix: "chore(deps-ci)"
12
+ groups:
13
+ github-actions:
14
+ patterns: ["*"]
15
+
16
+ - package-ecosystem: cargo
17
+ directory: /
18
+ schedule:
19
+ interval: weekly
20
+ commit-message:
21
+ prefix: "chore(deps-rust)"
22
+ groups:
23
+ rust-minor-patch:
24
+ update-types: ["minor", "patch"]
25
+
26
+ # `uv` is a first-class Dependabot ecosystem (understands pyproject.toml and uv.lock
27
+ # natively) — not `pip`, which reads pyproject.toml but doesn't maintain uv.lock.
28
+ - package-ecosystem: uv
29
+ directory: /
30
+ schedule:
31
+ interval: weekly
32
+ commit-message:
33
+ prefix: "chore(deps-py)"
34
+ groups:
35
+ python-minor-patch:
36
+ update-types: ["minor", "patch"]
@@ -0,0 +1,28 @@
1
+ ## Summary
2
+
3
+ Brief description of what this PR does and why.
4
+
5
+ ## Changes
6
+
7
+ - Change 1
8
+ - Change 2
9
+
10
+ ## Testing
11
+
12
+ Describe how you tested these changes.
13
+
14
+ ```bash
15
+ uv sync --locked
16
+ uv run --no-sync pytest -q
17
+ cargo test --no-default-features
18
+ cargo clippy --all-targets -- -D warnings
19
+ cargo fmt --check
20
+ ```
21
+
22
+ ## Docs
23
+
24
+ - [ ] `README.md` and `docs/` updated to match any behavior or API change in this PR.
25
+
26
+ ## Related Issues
27
+
28
+ Fixes #123
@@ -0,0 +1,191 @@
1
+ name: CI
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ pull_request:
7
+ # Manual trigger for debugging a workflow change without pushing a commit.
8
+ workflow_dispatch:
9
+
10
+ # Cancel the superseded run for the same branch when a new push lands:
11
+ # rapid follow-up commits stop burning runner minutes on an already-outdated run.
12
+ concurrency:
13
+ group: ${{ github.workflow }}-${{ github.ref }}
14
+ cancel-in-progress: true
15
+
16
+ # Least-privilege default: every job here reads and builds, none writes anything back to
17
+ # the repo or an external registry, so the top-level GITHUB_TOKEN needs no more than this.
18
+ permissions:
19
+ contents: read
20
+
21
+ env:
22
+ CARGO_TERM_COLOR: always
23
+
24
+ jobs:
25
+ lint:
26
+ # Runs once, not per matrix leg: Rust fmt/clippy is Python-version-agnostic, so the
27
+ # 3.10–3.14 matrix would buy five identical copies of the same signal.
28
+ runs-on: ubuntu-latest
29
+ steps:
30
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
31
+
32
+ - name: Install Rust
33
+ # The action installs with rustup's `--profile minimal`, which ships neither
34
+ # rustfmt nor clippy — this input is what makes the two steps below runnable.
35
+ uses: dtolnay/rust-toolchain@6bed0761d98439e5a578e2877258200ad565ba87 # stable
36
+ with:
37
+ components: rustfmt, clippy
38
+
39
+ - name: Cache cargo
40
+ # rust-cache, not sccache, in CI: every strategy pays its warm-up per job, and a
41
+ # ~10s clean build never amortizes sccache's install + cache-restore overhead on
42
+ # top of it. rust-cache already keys per job on the lockfile. sccache is wired
43
+ # where it does pay — the local edit/compile loop, via the Makefile (which does
44
+ # the same: sccache locally, no compiler cache in CI).
45
+ uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
46
+
47
+ - name: Cargo fmt
48
+ run: cargo fmt --check
49
+
50
+ - name: Cargo clippy
51
+ run: cargo clippy --all-targets -- -D warnings
52
+
53
+ - name: Cargo clippy (no default features)
54
+ # Second pass in the configuration `Cargo test` / `Compile benches` below actually
55
+ # build (--no-default-features): pyo3's cfg flags differ between the two configs,
56
+ # so each pass can surface lints in code the other never compiles.
57
+ run: cargo clippy --all-targets --no-default-features -- -D warnings
58
+
59
+ - name: Cargo deny (licenses, advisories, bans)
60
+ # The repo-wide, permanent licensing gate (spec v0.4 item 6): the
61
+ # permissive-only allowlist and the advisory/ban policy live in
62
+ # deny.toml; `make deny` runs the same three checks locally. The
63
+ # action ships a prebuilt cargo-deny binary, so the gate costs
64
+ # seconds, not the minutes a `cargo install cargo-deny --locked`
65
+ # would add to every run.
66
+ uses: EmbarkStudios/cargo-deny-action@3c6349835b2b7b196a839186cb8b78e02f7b5f25 # v2.1.1
67
+ with:
68
+ command-arguments: licenses advisories bans
69
+
70
+ - name: Install uv
71
+ # For the ruff step below: ruff is a dev-group dependency (pyproject.toml),
72
+ # so the lockfile — not this workflow — owns its version.
73
+ uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
74
+
75
+ - name: Ruff
76
+ # The Python-side lint gate (config in pyproject.toml [tool.ruff];
77
+ # `make lint` runs the same check locally). --no-install-project keeps
78
+ # this a pure lint step: only the dev deps are installed, never the
79
+ # tors wheel (whose maturin build is the test job's business — the
80
+ # Rust gates above already compile the code it would wrap).
81
+ run: |
82
+ uv sync --locked --no-install-project
83
+ uv run --no-sync ruff check .
84
+
85
+ test:
86
+ # Cheap signals first: a fmt or clippy
87
+ # failure surfaces from `lint` in one fast job instead of after five matrix legs have
88
+ # each spent a uv sync + extension build on the same break.
89
+ needs: [lint]
90
+ runs-on: ubuntu-latest
91
+ strategy:
92
+ fail-fast: false
93
+ matrix:
94
+ python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
95
+ steps:
96
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
97
+
98
+ - name: Install Rust
99
+ uses: dtolnay/rust-toolchain@6bed0761d98439e5a578e2877258200ad565ba87 # stable
100
+
101
+ - name: Cache cargo
102
+ # Same choice as the lint job's cache step (rationale there).
103
+ uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
104
+
105
+ - name: Cargo test
106
+ # extension-module doesn't link libpython, so plain `cargo test` needs it off.
107
+ # Rust-only; the crate isn't Python-version-specific, so this runs once per job
108
+ # rather than once per matrix entry.
109
+ if: matrix.python-version == '3.12'
110
+ run: cargo test --no-default-features
111
+
112
+ - name: Compile benches
113
+ # Compile-guard only: the bench docstring (benches/normalize.rs) and the design
114
+ # spec both promise that CI compiles the criterion benches; `--no-run` keeps that
115
+ # promise without spending runner time executing them. `--no-default-features`:
116
+ # same libpython-linking reason as `Cargo test` above — a bench binary links the rlib.
117
+ if: matrix.python-version == '3.12'
118
+ run: cargo bench --no-run --no-default-features
119
+
120
+ - name: Install uv
121
+ uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
122
+ with:
123
+ python-version: ${{ matrix.python-version }}
124
+
125
+ - name: Sync dependencies and build the extension
126
+ # --locked: fail if uv.lock is out of date with pyproject.toml, rather than silently
127
+ # rewriting it.
128
+ run: uv sync --locked
129
+
130
+ - name: Pytest
131
+ # -m "not timing": the timing lane (tests/test_gil_release.py and
132
+ # tests/test_performance.py — the asyncio heartbeat-gap and wall-band
133
+ # measurement cells, marked `timing` in pyproject.toml) is slow and
134
+ # load-sensitive, so the matrix legs deselect it and ONE dedicated
135
+ # step on this 3.12 leg runs it right after: the GIL/perf contracts
136
+ # still gate every push, once, instead of being paid on all five
137
+ # legs. The marker split is exact (both lane files carry a module-level
138
+ # pytestmark, nothing else does), so no test loses its lane — verified
139
+ # by `pytest --collect-only -m timing` + `-m "not timing"` summing to
140
+ # the unfiltered count.
141
+ # -rP: also show the captured stdout of PASSING tests. The suite's
142
+ # print-only measurement cells (deliberately unasserted bands, e.g.
143
+ # b64_decode's wall parity) record their numbers via print, which the
144
+ # default -q report swallows — -rP surfaces them in the CI log for
145
+ # the record.
146
+ run: uv run --no-sync pytest -q -rP -m "not timing"
147
+
148
+ - name: Pytest (timing lane)
149
+ # The lane the matrix legs above deselect, run ONCE (the 3.12 leg,
150
+ # after the main pytest step): the GIL-release and wall-time
151
+ # measurement cells. -rP so the print-only recorded bands (and the
152
+ # b64 battery's recorded gh-145264 divergences, should this leg's
153
+ # stdlib predate the fix) surface in the log. Runs last on the leg so
154
+ # a timing failure never masks the fast lane's results.
155
+ if: matrix.python-version == '3.12'
156
+ run: uv run --no-sync pytest -q -rP -m timing
157
+
158
+ fuzz-smoke:
159
+ # A cheap regression net, not a real fuzzing campaign: 15s per target
160
+ # (CONTRIBUTING.md's `make fuzz-quick` runs the same targets for 30s
161
+ # locally) — enough to catch a reintroduced panic on every push without
162
+ # meaningfully lengthening CI. A real multi-hour campaign against one
163
+ # target of active suspicion is a local `cargo +nightly fuzz run
164
+ # <target>`, not something this job runs.
165
+ runs-on: ubuntu-latest
166
+ steps:
167
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
168
+
169
+ - name: Install Rust (nightly)
170
+ # cargo-fuzz's instrumented builds need nightly's -Z sanitizer flags;
171
+ # the rest of CI stays on stable (see the lint job's own note).
172
+ uses: dtolnay/rust-toolchain@6bed0761d98439e5a578e2877258200ad565ba87 # stable
173
+ with:
174
+ toolchain: nightly
175
+
176
+ - name: Cache cargo
177
+ uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
178
+ with:
179
+ workspaces: ". -> target\nfuzz -> target"
180
+
181
+ - name: Install cargo-fuzz
182
+ run: cargo install cargo-fuzz --locked
183
+
184
+ - name: Fuzz (15s per target)
185
+ working-directory: fuzz
186
+ run: |
187
+ set -e
188
+ for t in decode_utf8 decode_utf16 b64_decode html_unescape fence chunk_hierarchical normalize search segmentation diff phonetic bm25 tfidf; do
189
+ echo "=== $t ==="
190
+ cargo +nightly fuzz run "$t" -- -max_total_time=15 -close_fd_mask=3
191
+ done
@@ -0,0 +1,39 @@
1
+ # Enforces Conventional Commits on the PR title, which is what release-please parses to
2
+ # decide the next version bump and CHANGELOG entry.
3
+
4
+ name: Commitlint
5
+
6
+ on:
7
+ pull_request:
8
+ types: [opened, edited, reopened, synchronize]
9
+
10
+ permissions:
11
+ contents: read
12
+ pull-requests: read
13
+
14
+ jobs:
15
+ commitlint:
16
+ runs-on: ubuntu-latest
17
+ timeout-minutes: 5
18
+ steps:
19
+ - uses: amannn/action-semantic-pull-request@48f256284bd46cdaab1048c3721360e808335d50 # v6.1.1
20
+ if: github.actor != 'dependabot[bot]' && github.actor != 'github-actions[bot]'
21
+ env:
22
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
23
+ with:
24
+ types: |
25
+ feat
26
+ fix
27
+ docs
28
+ style
29
+ refactor
30
+ perf
31
+ test
32
+ chore
33
+ ci
34
+ build
35
+ subjectPattern: ^[a-z].+
36
+ subjectPatternError: |
37
+ The subject "{subject}" found in the pull request title "{title}"
38
+ didn't match the configured pattern. Please ensure that the subject
39
+ starts with a lowercase letter.
@@ -0,0 +1,51 @@
1
+ # Builds the mkdocs site and deploys it to GitHub Pages via the modern OIDC-based deploy
2
+ # flow (actions/deploy-pages), not the legacy push-to-a-gh-pages-branch approach. Runs
3
+ # `mkdocs build --strict` so a broken nav entry or bad link fails CI instead of silently
4
+ # publishing a broken page.
5
+
6
+ name: docs
7
+
8
+ on:
9
+ push:
10
+ branches: [main]
11
+ paths:
12
+ - "docs/**"
13
+ - "mkdocs.yml"
14
+ - "pyproject.toml"
15
+ - "uv.lock"
16
+ - ".github/workflows/docs.yml"
17
+ workflow_dispatch: {}
18
+
19
+ permissions:
20
+ contents: read
21
+
22
+ jobs:
23
+ build:
24
+ runs-on: ubuntu-latest
25
+ steps:
26
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
27
+ - name: Install uv
28
+ uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
29
+ - name: Sync docs dependencies
30
+ # --locked: fail if uv.lock is out of date, rather than silently rewriting it.
31
+ run: uv sync --locked --group docs --no-install-project --no-default-groups
32
+ - name: Build
33
+ run: uv run --no-sync mkdocs build --strict
34
+ - uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d # v6
35
+ - uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0
36
+ with:
37
+ path: site
38
+
39
+ deploy:
40
+ needs: build
41
+ runs-on: ubuntu-latest
42
+ if: github.ref == 'refs/heads/main'
43
+ permissions:
44
+ pages: write
45
+ id-token: write # Required by actions/deploy-pages to verify the deployment source.
46
+ environment:
47
+ name: github-pages
48
+ url: ${{ steps.deployment.outputs.page_url }}
49
+ steps:
50
+ - id: deployment
51
+ uses: actions/deploy-pages@368f82528645a54fb793d4d04e342629a3f51346 # v5
@@ -0,0 +1,68 @@
1
+ # The scheduled deep fuzzing pass. ci.yml's `fuzz-smoke` job runs the same
2
+ # targets for 15s on every push (a regression net); this job gives them a
3
+ # longer 30s budget on a weekly schedule, when runner minutes are cheap and
4
+ # nobody is waiting on the result, plus a manual `workflow_dispatch` trigger
5
+ # for running a pass on demand (e.g. after a change to a covered core).
6
+ # Crashes fail the job (no continue-on-error); the crash inputs libFuzzer
7
+ # writes to fuzz/artifacts/ and the accumulated corpus are uploaded on
8
+ # failure only, so a red run carries its repro case with it.
9
+
10
+ name: fuzz
11
+
12
+ on:
13
+ schedule:
14
+ # Mondays 03:20 UTC: off the top of the hour, away from the busiest
15
+ # minute of GitHub's scheduled-workload peak.
16
+ - cron: "20 3 * * 1"
17
+ workflow_dispatch:
18
+
19
+ permissions:
20
+ contents: read
21
+
22
+ env:
23
+ CARGO_TERM_COLOR: always
24
+
25
+ jobs:
26
+ fuzz:
27
+ runs-on: ubuntu-latest
28
+ steps:
29
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
30
+
31
+ - name: Install Rust (nightly)
32
+ # cargo-fuzz's instrumented builds need nightly's -Z sanitizer flags;
33
+ # the rest of CI stays on stable (see the lint job's own note in
34
+ # ci.yml).
35
+ uses: dtolnay/rust-toolchain@6bed0761d98439e5a578e2877258200ad565ba87 # stable
36
+ with:
37
+ toolchain: nightly
38
+
39
+ - name: Cache cargo
40
+ # Same choice as ci.yml's fuzz-smoke job (rationale there).
41
+ uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
42
+ with:
43
+ workspaces: ". -> target\nfuzz -> target"
44
+
45
+ - name: Install cargo-fuzz
46
+ run: cargo install cargo-fuzz --locked
47
+
48
+ - name: Fuzz (30s per target)
49
+ working-directory: fuzz
50
+ run: |
51
+ set -e
52
+ for t in decode_utf8 decode_utf16 b64_decode html_unescape fence chunk_hierarchical normalize search segmentation diff phonetic bm25 tfidf; do
53
+ echo "=== $t ==="
54
+ cargo +nightly fuzz run "$t" -- -max_total_time=30 -close_fd_mask=3
55
+ done
56
+
57
+ - name: Upload corpus and crash artifacts
58
+ # Failure only: a green run's corpus is reproducible from the seed
59
+ # inputs, but a red run's crash file is the repro case the report
60
+ # needs (minimize it locally with `cargo fuzz tmin`).
61
+ if: failure()
62
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
63
+ with:
64
+ name: fuzz-crash-artifacts
65
+ path: |
66
+ fuzz/artifacts
67
+ fuzz/corpus
68
+ if-no-files-found: ignore
@@ -0,0 +1,162 @@
1
+ # Builds abi3 wheels (one per platform, valid for every Python 3.10+ per Cargo.toml's
2
+ # `abi3-py310` feature — no per-Python-version build matrix needed) plus an sdist, then
3
+ # publishes to PyPI via Trusted Publishing (OIDC) on a release-please tag. No PyPI API
4
+ # token exists anywhere in this repo — `id-token: write` is the whole auth story.
5
+ #
6
+ # Platform set is the mainstream cross-platform tier most serious PyO3 packages ship
7
+ # (Linux manylinux + musllinux on x86_64/aarch64, macOS x86_64/aarch64, Windows x64/arm64)
8
+ # — not maturin generate-ci's full exotic-target sprawl (32-bit x86, armv7, s390x, ppc64le,
9
+ # emscripten, android), which stays out until a real user asks for one of those.
10
+ #
11
+ # Tag-trigger caveat: release-please publishes its tag using GITHUB_TOKEN by default, and
12
+ # GitHub does not start workflow runs from GITHUB_TOKEN-authored events — so this workflow's
13
+ # `push: tags` trigger will not fire automatically unless release-please.yml is given a
14
+ # GitHub App token (see that workflow's header). Until one is configured, run this via
15
+ # `workflow_dispatch` against the new tag after a release PR merges.
16
+ #
17
+ # Job order is deliberately cheapest-first: wheels-linux (and sdist) run on ubuntu-22.04,
18
+ # by far the cheapest and fastest GitHub-hosted runner. wheels-other (macOS, Windows —
19
+ # both several times the per-minute cost of Linux, and macOS/Windows arm64 runners are
20
+ # scarcer capacity) only starts once every Linux leg has passed, via `needs`. A correctness
21
+ # bug that fails on Linux therefore never burns the expensive runners at all.
22
+
23
+ name: publish
24
+
25
+ on:
26
+ push:
27
+ tags: ["v[0-9]+.[0-9]+.[0-9]+"]
28
+ workflow_dispatch: {}
29
+
30
+ permissions:
31
+ contents: read
32
+
33
+ jobs:
34
+ wheels-linux:
35
+ strategy:
36
+ fail-fast: false
37
+ matrix:
38
+ include:
39
+ - target: x86_64
40
+ manylinux: auto
41
+ - target: aarch64
42
+ manylinux: auto
43
+ - target: x86_64
44
+ manylinux: musllinux_1_2
45
+ - target: aarch64
46
+ manylinux: musllinux_1_2
47
+ runs-on: ubuntu-22.04
48
+ steps:
49
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
50
+ - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
51
+ with:
52
+ python-version: "3.13"
53
+ - name: Build wheel
54
+ uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0
55
+ with:
56
+ target: ${{ matrix.target }}
57
+ args: --release --out dist
58
+ manylinux: ${{ matrix.manylinux }}
59
+ - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
60
+ with:
61
+ name: wheels-linux-${{ matrix.target }}-${{ matrix.manylinux }}
62
+ path: dist
63
+
64
+ sdist:
65
+ runs-on: ubuntu-latest
66
+ steps:
67
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
68
+ - name: Build sdist
69
+ uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0
70
+ with:
71
+ command: sdist
72
+ args: --out dist
73
+ - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
74
+ with:
75
+ name: sdist
76
+ path: dist
77
+
78
+ wheels-other:
79
+ needs: wheels-linux
80
+ strategy:
81
+ fail-fast: false
82
+ matrix:
83
+ include:
84
+ - os: macos-15-intel
85
+ target: x86_64
86
+ - os: macos-latest
87
+ target: aarch64
88
+ - os: windows-latest
89
+ target: x64
90
+ python_arch: x64
91
+ - os: windows-11-arm
92
+ target: aarch64
93
+ python_arch: arm64
94
+ runs-on: ${{ matrix.os }}
95
+ steps:
96
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
97
+ - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
98
+ with:
99
+ python-version: "3.13"
100
+ architecture: ${{ matrix.python_arch || 'x64' }}
101
+ - name: Build wheel
102
+ uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0
103
+ with:
104
+ target: ${{ matrix.target }}
105
+ args: --release --out dist
106
+ - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
107
+ with:
108
+ name: wheels-${{ matrix.os }}-${{ matrix.target }}
109
+ path: dist
110
+
111
+ publish:
112
+ name: Publish to PyPI
113
+ needs: [wheels-linux, wheels-other, sdist]
114
+ runs-on: ubuntu-latest
115
+ environment: pypi
116
+ permissions:
117
+ id-token: write # Trusted Publishing — no API token secret needed or used.
118
+ steps:
119
+ - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
120
+ with:
121
+ pattern: "wheels-*"
122
+ path: dist
123
+ merge-multiple: true
124
+ - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
125
+ with:
126
+ name: sdist
127
+ path: dist
128
+ - name: Install uv
129
+ uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
130
+ - name: Publish
131
+ run: uv publish --trusted-publishing automatic dist/*
132
+
133
+ publish-crate:
134
+ name: Publish to crates.io
135
+ # Independent of the PyPI leg above: the crate (published as `tors-core`, see
136
+ # Cargo.toml's own comment on the name collision) needs no wheel artifacts, just the
137
+ # source tree and Cargo.lock, so it runs in parallel rather than behind `needs`.
138
+ #
139
+ # First release only: crates.io has no PyPI-style "pending trusted publisher" for a
140
+ # crate that has never been published, so the very first `tors-core` release must be
141
+ # `cargo publish`d manually (from a machine already `cargo login`-ed). Only after that
142
+ # succeeds can a Trusted Publisher be configured in the crate's crates.io settings
143
+ # (GitHub owner `AZX-PBC-OSS`, repo `tors`, workflow `publish.yml`) — this job is that
144
+ # configuration's target and does nothing until it exists.
145
+ runs-on: ubuntu-latest
146
+ environment: crates-io
147
+ permissions:
148
+ id-token: write # Trusted Publishing (OIDC) — no API token secret needed or used.
149
+ steps:
150
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
151
+ - name: Install Rust
152
+ uses: dtolnay/rust-toolchain@6bed0761d98439e5a578e2877258200ad565ba87 # stable
153
+ - name: Authenticate with crates.io
154
+ id: auth
155
+ uses: rust-lang/crates-io-auth-action@c6f97d42243bad5fab37ca0427f495c86d5b1a18 # v1.0.5
156
+ - name: Publish
157
+ # --no-default-features: `extension-module` (the crate's default) doesn't link
158
+ # libpython, which breaks a plain `cargo publish` verification build the same way
159
+ # it breaks `cargo test`/`cargo build` — see Cargo.toml's own feature comment.
160
+ env:
161
+ CARGO_REGISTRY_TOKEN: ${{ steps.auth.outputs.token }}
162
+ run: cargo publish -p tors-core --no-default-features