sentencex 1.0.2__tar.gz → 1.0.4__tar.gz

{sentencex-1.0.2 → sentencex-1.0.4}/Cargo.lock

@@ -653,7 +653,7 @@ checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73"
 
 [[package]]
 name = "sentencex"
-version = "0.1.2"
+version = "0.1.4"
 dependencies = [
  "clap",
  "criterion",
@@ -666,7 +666,7 @@ dependencies = [
 
 [[package]]
 name = "sentencex-js"
-version = "1.0.2"
+version = "1.0.4"
 dependencies = [
  "neon",
  "neon-build",
@@ -675,7 +675,7 @@ dependencies = [
 
 [[package]]
 name = "sentencex-py"
-version = "0.1.1"
+version = "0.1.4"
 dependencies = [
  "pyo3",
  "sentencex",
@@ -683,7 +683,7 @@ dependencies = [
 
 [[package]]
 name = "sentencex-wasm"
-version = "0.1.3"
+version = "0.1.4"
 dependencies = [
  "sentencex",
  "serde",

{sentencex-1.0.2 → sentencex-1.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sentencex
-Version: 1.0.2
+Version: 1.0.4
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research
 Classifier: Topic :: Text Processing

{sentencex-1.0.2 → sentencex-1.0.4}/bindings/python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "sentencex-py"
-version = "0.1.1"
+version = "0.1.4"
 edition = "2024"
 description = "Sentence segmentation library with wide language support optimized for speed and utility."
 authors = ["Santhosh Thottingal <santhosh.thottingal@gmail.com>"]

sentencex-1.0.4/bindings/python/publish.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+# Build and publish at pypi
+uv tool run maturin build -i python3.12
+uv tool run maturin build -i python3.13
+uv tool run maturin build -i python3.14
+uv tool run maturin publish

sentencex-1.0.4/paris.txt

@@ -0,0 +1,59 @@
+
+	Paris (
+	French pronunciation:
+	
+	
+		[paʁi]
+	
+	
+		 (
+		
+			
+				
+					
+						
+							
+						
+					
+					 
+				
+				listen
+			
+		)) Another sentence
+	
+		
+			
+		
+	ends here.
+[
+  '\n' +
+    '\tParis (\n' +
+    '\tFrench pronunciation:\n' +
+    '\t\n' +
+    '\t\n' +
+    '\t\t[paʁi]\n' +
+    '\t\n' +
+    '\t\n' +
+    '\t\t (\n' +
+    '\t\t\n' +
+    '\t\t\t\n' +
+    '\t\t\t\t\n' +
+    '\t\t\t\t\t\n' +
+    '\t\t\t\t\t\t\n' +
+    '\t\t\t\t\t\t\t\n' +
+    '\t\t\t\t\t\t\n' +
+    '\t\t\t\t\t\n' +
+    '\t\t\t\t\t \n' +
+    '\t\t\t\t\n' +
+    '\t\t\t\tlisten\n' +
+    '\t\t\t\n' +
+    '\t\t)) Another sentence\n' +
+    '\t\n' +
+    '\t\t\n' +
+    '\t\t\t\n' +
+    '\t\t\n' +
+    '\tends here.'
+]
+<p id="mwEA"><span class="cx-segment" data-segmentid="0"><b id="mwEQ">Paris</b> (<small about="#mwt16" data-mw="{&#34;parts&#34;:[{&#34;template&#34;:{&#34;target&#34;:{&#34;wt&#34;:&#34;IPA-fr&#34;,&#34;href&#34;:&#34;./Template:IPA-fr&#34;},&#34;params&#34;:{&#34;1&#34;:{&#34;wt&#34;:&#34;paʁi&#34;},&#34;3&#34;:{&#34;wt&#34;:&#34;Paris1.ogg&#34;}},&#34;i&#34;:0}}]}" id="mwEg" typeof="mw:Transclusion">French pronunciation:</small><span about="#mwt16" class="IPA" title="Representation in the International Phonetic Alphabet (IPA)"><a class="cx-link" data-linkid="1" href="./Help:IPA/French" rel="mw:WikiLink" title="Help:IPA/French">[paʁi]</a></span><small about="#mwt16" class="nowrap" id="mwEw"><span typeof="mw:Entity"> </span>(<span class="unicode haudio"><span class="fn"><span style="white-space:nowrap"><span data-mw="{&#34;caption&#34;:&#34;About this sound&#34;}" typeof="mw:Image"><a href="./File:Paris1.ogg"><img data-file-height="20" data-file-type="drawing" data-file-width="20" height="11" resource="./File:Loudspeaker.svg" src="//upload.wikimedia.org/wikipedia/commons/thumb/8/8a/Loudspeaker.svg/11px-Loudspeaker.svg.png" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/8/8a/Loudspeaker.svg/22px-Loudspeaker.svg.png 2x, //upload.wikimedia.org/wikipedia/commons/thumb/8/8a/Loudspeaker.svg/17px-Loudspeaker.svg.png 1.5x" width="11"></img></a></span><span typeof="mw:Entity"> </span></span><a href="//upload.wikimedia.org/wikipedia/commons/2/2c/Paris1.ogg" rel="mw:MediaLink" title="Paris1.ogg">listen</a></span></span>)</small>) Another sentence<span data-mw="{&#34;caption&#34;:&#34;A different caption&#34;}" typeof="mw:Image"><a href="./File:Paris1232.ogg"><img data-file-height="20" data-file-type="drawing" data-file-width="20" height="11" resource="./File:Loudspeaker.svg" src="//upload.wikimedia.org/wikipedia/commons/thumb/8/8a/Loudspeaker.svg/11px-Loudspeaker.svg.png" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/8/8a/Loudspeaker.svg/22px-Loudspeaker.svg.png 2x, //upload.wikimedia.org/wikipedia/commons/thumb/8/8a/Loudspeaker.svg/17px-Loudspeaker.svg.png 1.5x" width="11"></img></a></span>ends here.</span></p>
+==Categories==
+[]

{sentencex-1.0.2 → sentencex-1.0.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sentencex"
-version = "1.0.2"
+version = "1.0.4"
 requires-python = ">=3.10"
 description = "Sentence segmenter that supports ~300 languages"
 authors = [{name = "Santhosh Thottingal", email = "santhosh.thottingal@gmail.com"}]

{sentencex-1.0.2 → sentencex-1.0.4}/src/constants.rs

@@ -26,7 +26,7 @@ pub fn get_quote_pairs() -> HashMap<&'static str, &'static str> {
 lazy_static::lazy_static! {
     pub static ref PARENS_REGEX: Regex = Regex::new(r"[\(（<{\[](?:[^\)\]}>）]|\\[\)\]}>）])*[\)\]}>）]").unwrap();
     pub static ref EMAIL_REGEX: Regex = Regex::new(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,7}").unwrap();
-    pub static ref NUMBERED_REFERENCE_REGEX: Regex = Regex::new(r"^ ?(\[\d+])+").unwrap();
+    pub static ref NUMBERED_REFERENCE_REGEX: Regex = Regex::new(r"^(\s*\[\d+])+").unwrap();
     pub static ref SPACE_AFTER_SEPARATOR: Regex = Regex::new(r"^\s+").unwrap();
     pub static ref QUOTES_REGEX: Regex = {
         let quote_pairs = get_quote_pairs();

{sentencex-1.0.2 → sentencex-1.0.4}/src/languages/language.rs

@@ -42,9 +42,8 @@ pub trait Language {
         let mut boundaries = Vec::with_capacity(estimated_sentences);
 
         // Split by paragraph breaks (one or more newlines with optional whitespace)
-        let para_split_re = Regex::new(r"\n[\r\s]*\n").unwrap();
+        let para_split_re = Regex::new(r"\n[\r]*\n").unwrap();
         let paragraphs: Vec<&str> = para_split_re.split(text).collect();
-
         // Pre-calculate all paragraph offsets in one pass
         let mut paragraph_offsets = Vec::with_capacity(paragraphs.len());
         let mut current_offset = 0;

{sentencex-1.0.2 → sentencex-1.0.4}/src/lib.rs

@@ -107,7 +107,7 @@ fn chunk_text(text: &str, chunk_size: usize) -> Vec<&str> {
     let mut chunks = Vec::new();
 
     // Split by paragraph breaks (one or more newlines with optional whitespace)
-    let re = Regex::new(r"\n[\r\s]*\n").unwrap();
+    let re = Regex::new(r"\n[\r]*\n").unwrap();
 
     // Get paragraph parts and their positions
     let mut paragraphs = Vec::new();

{sentencex-1.0.2 → sentencex-1.0.4}/src/main.rs

@@ -59,7 +59,7 @@ fn main() {
         let sentences = segment(&cli.language, &text);
         let elapsed = start_time.elapsed();
         for sentence in sentences.iter() {
-            println!("{}", sentence);
+            println!("* {}", sentence);
         }
 
         eprintln!("Time taken for segment(): {:?}", elapsed);

{sentencex-1.0.2 → sentencex-1.0.4}/tests/en.txt

@@ -259,6 +259,11 @@ Hydrogen is a gas. [1] It is colorless, odorless, tasteless and highly flammable
 Hydrogen is a gas. [1]
 It is colorless, odorless, tasteless and highly flammable
 ===
+Hydrogen is a gas. [1][2] [3] It is colorless, odorless, tasteless and highly flammable
+---
+Hydrogen is a gas. [1][2] [3]
+It is colorless, odorless, tasteless and highly flammable
+===
 This function (see. section 4.2) is important. Let's continue.
 ---
 This function (see. section 4.2) is important.