unicode-segmentation-rs 0.3.2__tar.gz → 0.3.3__tar.gz

{unicode_segmentation_rs-0.3.2 → unicode_segmentation_rs-0.3.3}/.github/actions/pre-commit-setup/action.yml

@@ -2,6 +2,8 @@
 #
 # SPDX-License-Identifier: CC0-1.0
 
+# This file is maintained in https://github.com/WeblateOrg/meta/
+
 name: pre-commit setup
 description: Prepare pre-commit environment
 
@@ -19,8 +21,7 @@ runs:
   - name: Setup Python
     uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97   # v7.0.0
     with:
-      python-version: '3.15'
-      allow-prereleases: true
+      python-version: '3.14'
   - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9   # v9.0.0
     with:
-      version: 0.12.1
+      version: 0.12.3

{unicode_segmentation_rs-0.3.2 → unicode_segmentation_rs-0.3.3}/.github/workflows/ci.yml

@@ -53,7 +53,7 @@ jobs:
       with:
         save-cache: ${{ github.ref == 'refs/heads/main' }}
         cache-suffix: ${{ steps.setup_python.outputs.python-version }}
-        version: 0.12.1
+        version: 0.12.3
         activate-environment: 'true'
     - run: uv sync
     - name: Run tests
@@ -315,7 +315,7 @@ jobs:
         persist-credentials: false
     - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
     - name: Generate artifact attestation
-      uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4.1.1
+      uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2
       with:
         subject-path: wheels-*/*
     - name: Publish to PyPI

{unicode_segmentation_rs-0.3.2 → unicode_segmentation_rs-0.3.3}/.github/workflows/pre-commit.yml

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: CC0-1.0
 
 # This file is maintained in https://github.com/WeblateOrg/meta/
+
 name: Pre-commit check
 
 on:

unicode_segmentation_rs-0.3.3/.github/workflows/security-audit.yml

@@ -0,0 +1,42 @@
+# Copyright © Michal Čihař <michal@weblate.org>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+name: Security audit
+
+on:
+  pull_request:
+  push:
+    branches:
+    - main
+    paths:
+    - Cargo.lock
+    - Cargo.toml
+    - .github/workflows/security-audit.yml
+  schedule:
+  - cron: 37 4 * * 1
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+env:
+  # renovate: datasource=crate depName=cargo-audit
+  CARGO_AUDIT_VERSION: 0.22.2
+
+jobs:
+  rustsec:
+    name: RustSec audit
+    runs-on: ubuntu-24.04
+    timeout-minutes: 15
+    steps:
+    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      with:
+        persist-credentials: false
+    - name: Install cargo-audit
+      uses: taiki-e/install-action@6c6fd71fe4fb72c3697d269963d0e15df8adedad # v2.85.10
+      with:
+        fallback: none
+        tool: cargo-audit@${{ env.CARGO_AUDIT_VERSION }}
+    - name: Audit Rust dependencies
+      run: cargo audit --file Cargo.lock

{unicode_segmentation_rs-0.3.2 → unicode_segmentation_rs-0.3.3}/.pre-commit-config.yaml

@@ -21,7 +21,7 @@ repos:
   hooks:
   - id: yamllint
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.16.1
+  rev: v0.16.2
   hooks:
   - id: ruff-check
     args:
@@ -52,7 +52,7 @@ repos:
   hooks:
   - id: zizmor
 - repo: https://github.com/mongodb/kingfisher
-  rev: v1.110.0
+  rev: v1.112.0
   hooks:
   - id: kingfisher-auto
 - repo: https://github.com/rvben/rumdl-pre-commit

{unicode_segmentation_rs-0.3.2 → unicode_segmentation_rs-0.3.3}/Cargo.lock

@@ -37,9 +37,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3"
-version = "0.29.1"
+version = "0.29.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fc153f5fd745cc038b5eed86622125969f8a39834a57bc96beaaf2512b1da729"
+checksum = "4688ddedf473e32662b9b067670129a8afb8c18e351482c70d62ba4a88171e8b"
 dependencies = [
  "libc",
  "once_cell",
@@ -51,18 +51,18 @@ dependencies = [
 
 [[package]]
 name = "pyo3-build-config"
-version = "0.29.1"
+version = "0.29.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bb77d9aa6d647507b55c69ee714d266d84c526c78ff0bc6dd8757f58591e64d1"
+checksum = "f41027e41b4bd03f6e60f9f417fe24a6341a6bb744edd62b6f709f2a52ea30e9"
 dependencies = [
  "target-lexicon",
 ]
 
 [[package]]
 name = "pyo3-ffi"
-version = "0.29.1"
+version = "0.29.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3160087aa5733bce7d9a729f8c55f85a2a53b74742a09613178eeebff0722253"
+checksum = "e591a95526fead067432c3b3a33fc74770b87b1e04e73671090d9c2055a2b327"
 dependencies = [
  "libc",
  "pyo3-build-config",
@@ -70,9 +70,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3-macros"
-version = "0.29.1"
+version = "0.29.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "91f9d455db760a9a0b0ddeaac25f1390b8a36ba73dfbda9f127cac6fc340d4d5"
+checksum = "73225868fc1cd84eef2c3c230ddb91273bf1de46aeb8a4248da76d32a0924a1c"
 dependencies = [
  "proc-macro2",
  "pyo3-macros-backend",
@@ -82,9 +82,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3-macros-backend"
-version = "0.29.1"
+version = "0.29.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e343bcec300ff262f5806a33a4e51b6d097a8a46435f512fcb83e95592581625"
+checksum = "571575aa3749fa6216757dd47d2a3e7ef360f329a40f0666a9fbd14889024952"
 dependencies = [
  "heck",
  "proc-macro2",
@@ -138,7 +138,7 @@ checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8"
 
 [[package]]
 name = "unicode-segmentation-rs"
-version = "0.3.2"
+version = "0.3.3"
 dependencies = [
  "pyo3",
  "unicode-linebreak",

{unicode_segmentation_rs-0.3.2 → unicode_segmentation_rs-0.3.3}/Cargo.toml

@@ -1,5 +1,5 @@
 [dependencies]
-pyo3 = {version = "0.29.1", features = ["extension-module", "abi3-py310"]}
+pyo3 = {version = "0.29.2", features = ["extension-module", "abi3-py310"]}
 unicode-linebreak = "0.1.5"
 unicode-segmentation = "1.13.3"
 unicode-width = "0.2.2"
@@ -19,4 +19,4 @@ license = "Apache-2.0"
 name = "unicode-segmentation-rs"
 readme = "README.md"
 repository = "https://github.com/WeblateOrg/unicode-segmentation-rs"
-version = "0.3.2"
+version = "0.3.3"

{unicode_segmentation_rs-0.3.2 → unicode_segmentation_rs-0.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: unicode-segmentation-rs
-Version: 0.3.2
+Version: 0.3.3
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
@@ -26,6 +26,11 @@ Project-URL: Issue Tracker, https://github.com/WeblateOrg/unicode-segmentation-r
 Project-URL: Source Code, https://github.com/WeblateOrg/unicode-segmentation-rs
 Project-URL: Twitter, https://twitter.com/WeblateOrg
 
+<a href="https://weblate.org/"><img alt="Weblate" src="https://s.weblate.org/cdn/Logo-Darktext-borders.png" height="80px" /></a>
+
+**Weblate is libre software web-based continuous localization system,
+used by over 2500 libre projects and companies in more than 165 countries.**
+
 # unicode-segmentation-rs
 
 Python bindings for the Rust [unicode-segmentation](https://docs.rs/unicode-segmentation/), [unicode-linebreak](https://docs.rs/unicode-linebreak/), and [unicode-width](https://docs.rs/unicode-width/) crates, providing Unicode text segmentation and width calculation according to Unicode standards.
@@ -235,6 +240,9 @@ print(lines)
 
 ## API Reference
 
+Functions returning lists materialize the complete result. Applications processing very large or
+untrusted text should enforce an input-size limit appropriate for their environment.
+
 ### `graphemes(text: str, is_extended: bool) -> list[str]`
 
 Split a string into grapheme clusters. Set `is_extended=True` for extended grapheme clusters (recommended).

{unicode_segmentation_rs-0.3.2 → unicode_segmentation_rs-0.3.3}/README.md

@@ -1,3 +1,8 @@
+<a href="https://weblate.org/"><img alt="Weblate" src="https://s.weblate.org/cdn/Logo-Darktext-borders.png" height="80px" /></a>
+
+**Weblate is libre software web-based continuous localization system,
+used by over 2500 libre projects and companies in more than 165 countries.**
+
 # unicode-segmentation-rs
 
 Python bindings for the Rust [unicode-segmentation](https://docs.rs/unicode-segmentation/), [unicode-linebreak](https://docs.rs/unicode-linebreak/), and [unicode-width](https://docs.rs/unicode-width/) crates, providing Unicode text segmentation and width calculation according to Unicode standards.
@@ -207,6 +212,9 @@ print(lines)
 
 ## API Reference
 
+Functions returning lists materialize the complete result. Applications processing very large or
+untrusted text should enforce an input-size limit appropriate for their environment.
+
 ### `graphemes(text: str, is_extended: bool) -> list[str]`
 
 Split a string into grapheme clusters. Set `is_extended=True` for extended grapheme clusters (recommended).

{unicode_segmentation_rs-0.3.2 → unicode_segmentation_rs-0.3.3}/src/lib.rs

@@ -4,6 +4,8 @@
 
 #[pyo3::pymodule(gil_used = false)]
 mod unicode_segmentation_rs {
+    use std::ops::Range;
+
     use pyo3::prelude::*;
     use pyo3::types::PyList;
     use unicode_linebreak::{BreakOpportunity, linebreaks as unicode_linebreaks};
@@ -57,14 +59,16 @@ mod unicode_segmentation_rs {
 
     /// Get Unicode line-break opportunities as UTF-8 byte offsets.
     #[pyfunction]
-    fn line_breaks(text: &str) -> PyResult<Vec<(usize, bool)>> {
+    fn line_breaks<'py>(py: Python<'py>, text: &str) -> PyResult<Bound<'py, PyList>> {
         if text.is_empty() {
-            return Ok(vec![]);
+            return Ok(PyList::empty(py));
         }
 
-        Ok(unicode_linebreaks(text)
-            .map(|(offset, opportunity)| (offset, opportunity == BreakOpportunity::Mandatory))
-            .collect())
+        PyList::new(
+            py,
+            unicode_linebreaks(text)
+                .map(|(offset, opportunity)| (offset, opportunity == BreakOpportunity::Mandatory)),
+        )
     }
 
     /// Split a string at every Unicode line-break opportunity.
@@ -100,21 +104,25 @@ mod unicode_segmentation_rs {
     /// - Handles CJK characters with proper width calculation
     /// - Breaks long words only when necessary
     #[pyfunction]
-    fn gettext_wrap(text: &str, width: usize) -> PyResult<Vec<String>> {
+    fn gettext_wrap<'py>(
+        py: Python<'py>,
+        text: &str,
+        width: usize,
+    ) -> PyResult<Bound<'py, PyList>> {
         if text.is_empty() || width == 0 {
             return if text.is_empty() {
-                Ok(vec![])
+                Ok(PyList::empty(py))
             } else {
-                Ok(vec![text.to_string()])
+                PyList::new(py, [text])
             };
         }
 
-        Ok(wrap_po_chunks(PoChunks::new(text), width))
+        PyList::new(py, wrap_po_chunks(text, PoChunks::new(text), width))
     }
 
-    /// Iterator over borrowed chunks split using word boundaries and PO-specific rules.
+    /// Iterator over chunk byte ranges split using word boundaries and PO-specific rules.
     struct PoChunks<'a> {
-        text: &'a str,
+        text_len: usize,
         word_bounds: UWordBoundIndices<'a>,
         chunk_start: Option<usize>,
         last_char: Option<char>,
@@ -124,7 +132,7 @@ mod unicode_segmentation_rs {
     impl<'a> PoChunks<'a> {
         fn new(text: &'a str) -> Self {
             Self {
-                text,
+                text_len: text.len(),
                 word_bounds: text.split_word_bound_indices(),
                 chunk_start: None,
                 last_char: None,
@@ -133,11 +141,7 @@ mod unicode_segmentation_rs {
         }
 
         /// Add a word-boundary segment and return the completed chunk's byte range, if any.
-        fn add_segment(
-            &mut self,
-            segment_start: usize,
-            segment: &str,
-        ) -> Option<std::ops::Range<usize>> {
+        fn add_segment(&mut self, segment_start: usize, segment: &str) -> Option<Range<usize>> {
             let first_char = segment.chars().next().unwrap();
             let should_merge = self.last_char.is_some_and(|last_char| {
                 (self.second_last_char.is_none()
@@ -176,8 +180,8 @@ mod unicode_segmentation_rs {
         }
     }
 
-    impl<'a> Iterator for PoChunks<'a> {
-        type Item = &'a str;
+    impl Iterator for PoChunks<'_> {
+        type Item = Range<usize>;
 
         fn next(&mut self) -> Option<Self::Item> {
             while let Some((segment_start, segment)) = self.word_bounds.next() {
@@ -196,48 +200,100 @@ mod unicode_segmentation_rs {
                         );
                     }
 
-                    return Some(&self.text[completed_start..segment_end]);
+                    return Some(completed_start..segment_end);
                 }
 
                 if let Some(completed) = self.add_segment(segment_start, segment) {
-                    return Some(&self.text[completed]);
+                    return Some(completed);
                 }
             }
 
-            self.chunk_start
-                .take()
-                .map(|start| &self.text[start..self.text.len()])
+            self.chunk_start.take().map(|start| start..self.text_len)
         }
     }
 
-    /// Wrap borrowed chunks into lines respecting the width limit.
-    fn wrap_po_chunks<'a>(chunks: impl IntoIterator<Item = &'a str>, width: usize) -> Vec<String> {
-        let mut lines = Vec::new();
-        let mut current_line = String::new();
-        let mut current_width = 0;
+    /// Iterator that wraps chunk ranges into borrowed lines respecting the width limit.
+    struct WrappedLines<'a, I>
+    where
+        I: Iterator<Item = Range<usize>>,
+    {
+        text: &'a str,
+        chunks: I,
+        width: usize,
+        current_start: Option<usize>,
+        current_end: usize,
+        current_width: usize,
+        emit_current: bool,
+    }
 
-        for chunk in chunks {
-            let chunk_width: usize = chunk.width();
+    impl<'a, I> Iterator for WrappedLines<'a, I>
+    where
+        I: Iterator<Item = Range<usize>>,
+    {
+        type Item = &'a str;
 
-            if current_width + chunk_width > width && !current_line.is_empty() {
-                lines.push(std::mem::take(&mut current_line));
-                current_width = 0;
+        fn next(&mut self) -> Option<Self::Item> {
+            if self.emit_current {
+                self.emit_current = false;
+                self.current_width = 0;
+                let start = self.current_start.take()?;
+                return Some(&self.text[start..self.current_end]);
             }
-            current_line.push_str(chunk);
-            current_width += chunk_width;
 
-            // Force break on \n
-            if chunk.ends_with("\\n") {
-                lines.push(std::mem::take(&mut current_line));
-                current_width = 0;
+            for chunk_range in self.chunks.by_ref() {
+                let chunk_start = chunk_range.start;
+                let chunk_end = chunk_range.end;
+                debug_assert_eq!(
+                    self.current_end, chunk_start,
+                    "wrapped chunks should be contiguous"
+                );
+                let chunk = &self.text[chunk_range];
+                let chunk_width = chunk.width();
+
+                if self.current_width + chunk_width > self.width {
+                    if let Some(completed_start) = self.current_start.replace(chunk_start) {
+                        let completed_end = self.current_end;
+                        self.current_end = chunk_end;
+                        self.current_width = chunk_width;
+                        self.emit_current = chunk.ends_with("\\n");
+                        return Some(&self.text[completed_start..completed_end]);
+                    }
+                }
+
+                if self.current_start.is_none() {
+                    self.current_start = Some(chunk_start);
+                }
+                self.current_end = chunk_end;
+                self.current_width += chunk_width;
+
+                // Force break on \n
+                if chunk.ends_with("\\n") {
+                    self.current_width = 0;
+                    let start = self.current_start.take()?;
+                    return Some(&self.text[start..self.current_end]);
+                }
             }
-        }
 
-        if !current_line.is_empty() {
-            lines.push(current_line);
+            self.current_width = 0;
+            self.current_start
+                .take()
+                .map(|start| &self.text[start..self.current_end])
         }
+    }
 
-        lines
+    fn wrap_po_chunks<C>(text: &str, chunks: C, width: usize) -> WrappedLines<'_, C::IntoIter>
+    where
+        C: IntoIterator<Item = Range<usize>>,
+    {
+        WrappedLines {
+            text,
+            chunks: chunks.into_iter(),
+            width,
+            current_start: None,
+            current_end: 0,
+            current_width: 0,
+            emit_current: false,
+        }
     }
 
     /// Check if a string contains only mergeable characters
@@ -300,4 +356,53 @@ mod unicode_segmentation_rs {
                 | '~'
         )
     }
+
+    #[cfg(test)]
+    mod tests {
+        use std::cell::Cell;
+
+        use super::wrap_po_chunks;
+
+        #[test]
+        fn wrapped_lines_are_produced_incrementally() {
+            let consumed = Cell::new(0);
+            let text = "first second third fourth";
+            let chunks = [0..6, 6..13, 13..19, 19..25]
+                .into_iter()
+                .inspect(|_| consumed.set(consumed.get() + 1));
+            let mut lines = wrap_po_chunks(text, chunks, 7);
+
+            assert_eq!(lines.next(), Some("first "));
+            assert_eq!(consumed.get(), 2);
+            assert_eq!(lines.collect::<Vec<_>>(), ["second ", "third ", "fourth"]);
+            assert_eq!(consumed.get(), 4);
+        }
+
+        #[test]
+        fn wrapped_lines_preserve_edge_cases() {
+            let text = "first \\nlast";
+            assert_eq!(
+                wrap_po_chunks(text, [0..6, 6..8, 8..12], 6).collect::<Vec<_>>(),
+                ["first ", "\\n", "last"]
+            );
+            assert!(
+                wrap_po_chunks("", std::iter::empty(), 10)
+                    .collect::<Vec<_>>()
+                    .is_empty()
+            );
+            assert_eq!(
+                wrap_po_chunks("unwrapped", std::iter::once(0..9), 0).collect::<Vec<_>>(),
+                ["unwrapped"]
+            );
+        }
+
+        #[test]
+        fn wrapped_lines_borrow_from_the_input() {
+            let text = "first second";
+            let line = wrap_po_chunks(text, [0..6, 6..12], 20).next().unwrap();
+
+            assert_eq!(line, text);
+            assert_eq!(line.as_ptr(), text.as_ptr());
+        }
+    }
 }