blitztext 0.1.0__cp38-none-win32.whl

blitztext/__init__.py

@@ -0,0 +1,5 @@
+from .blitztext import *
+
+__doc__ = blitztext.__doc__
+if hasattr(blitztext, "__all__"):
+    __all__ = blitztext.__all__

blitztext/__init__.pyi

@@ -0,0 +1,159 @@
+from typing import Dict, List, Optional, Tuple
+
+class KeywordMatch:
+    """
+    Represents a matched keyword in the text.
+
+    Attributes:
+        keyword (str): The matched keyword or its clean name.
+        similarity (float): The similarity score of the match (1.0 for exact matches).
+        start (int): The starting index of the match in the text.
+        end (int): The ending index of the match in the text.
+    """
+    keyword: str
+    similarity: float
+    start: int
+    end: int
+
+    def __repr__(self) -> str:
+        """
+        Returns a string representation of the KeywordMatch object.
+        """
+        ...
+
+class KeywordProcessor:
+    """
+    A processor for efficient keyword matching and replacement in text.
+
+    Based on AhoCorasick for fast keyword operations,
+    including fuzzy matching capabilities.
+    """
+
+    def __init__(self, case_sensitive: Optional[bool] = None, allow_overlaps: Optional[bool] = None) -> None:
+        """
+        Initialize a new KeywordProcessor.
+
+        Args:
+            case_sensitive: If True, matching will be case-sensitive. Default is False.
+            allow_overlaps: If True, allows overlapping matches. Default is True.
+        """
+        ...
+
+    def add_keyword(self, keyword: str, clean_name: Optional[str] = None) -> None:
+        """
+        Add a keyword to the processor.
+
+        Args:
+            keyword: The keyword to add.
+            clean_name: An optional 'clean' version of the keyword to return when found.
+        """
+        ...
+
+    def remove_keyword(self, keyword: str) -> bool:
+        """
+        Remove a keyword from the processor.
+
+        Args:
+            keyword: The keyword to remove.
+
+        Returns:
+            True if the keyword was found and removed, False otherwise.
+        """
+        ...
+
+    def extract_keywords(self, text: str, threshold: Optional[float] = None) -> List[KeywordMatch]:
+        """
+        Extract keywords from the given text.
+
+        Args:
+            text: The text to search for keywords.
+            threshold: Threshold for fuzzy matching (0.0 to 1.0). Default is 1.0 (exact matches only).
+
+        Returns:
+            A list of KeywordMatch objects representing the found keywords.
+        """
+        ...
+
+    def parallel_extract_keywords_from_texts(self, texts: List[str], threshold: Optional[float] = None) -> List[List[KeywordMatch]]:
+        """
+        Extract keywords from multiple texts in parallel.
+
+        Args:
+            texts: A list of texts to search for keywords.
+            threshold: Threshold for fuzzy matching (0.0 to 1.0). Default is 1.0 (exact matches only).
+
+        Returns:
+            A list of lists of KeywordMatch objects, one list per input text.
+        """
+        ...
+
+    def replace_keywords(self, text: str, threshold: Optional[float] = None) -> str:
+        """
+        Replace keywords in the given text with their clean names.
+
+        Args:
+            text: The text in which to replace keywords.
+            threshold: Threshold for fuzzy matching (0.0 to 1.0). Default is 1.0 (exact matches only).
+
+        Returns:
+            The text with keywords replaced by their clean names.
+        """
+        ...
+
+    def parallel_replace_keywords_from_texts(self, texts: List[str], threshold: Optional[float] = None) -> List[str]:
+        """
+        Replace keywords in multiple texts in parallel.
+
+        Args:
+            texts: A list of texts in which to replace keywords.
+            threshold: Threshold for fuzzy matching (0.0 to 1.0). Default is 1.0 (exact matches only).
+
+        Returns:
+            A list of strings with keywords replaced by their clean names.
+        """
+        ...
+
+    def get_all_keywords(self) -> Dict[str, str]:
+        """
+        Get all keywords and their clean names.
+
+        Returns:
+            A dictionary where keys are keywords and values are their clean names.
+        """
+        ...
+
+    def set_non_word_boundaries(self, boundaries: List[str]) -> None:
+        """
+        Set the characters that are considered part of a word.
+
+        Args:
+            boundaries: A list of characters to be set as non-word boundaries.
+        """
+        ...
+
+    def add_non_word_boundary(self, boundary: str) -> None:
+        """
+        Add a single character as a non-word boundary.
+
+        Args:
+            boundary: The character to be added as a non-word boundary.
+        """
+        ...
+
+    def __len__(self) -> int:
+        """
+        Get the number of keywords in the processor.
+
+        Returns:
+            The number of keywords.
+        """
+        ...
+
+    def __repr__(self) -> str:
+        """
+        Get a string representation of the KeywordProcessor.
+
+        Returns:
+            A string representation including the number of keywords.
+        """
+        ...

blitztext-0.1.0.dist-info/METADATA

@@ -0,0 +1,282 @@
+Metadata-Version: 2.3
+Name: blitztext
+Version: 0.1.0
+Classifier: Programming Language :: Rust
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+License-File: LICENSE
+Summary: A library for fast keywords extraction and replacement in strings.
+Keywords: keyword,search,fuzzy,trie,aho-corasick
+Author: Praise Oketola <oketola.praise@gmail.com>
+Author-email: Praise Oketola <oketola.praise@gmail.com>
+License: MIT
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+Project-URL: Source Code, https://github.com/praise2112/blitztext
+
+# BlitzText
+
+BlitzText is a high-performance library for efficient keyword extraction and replacement in strings. It is based on the [FlashText](https://github.com/vi3k6i5/flashtext) and  [Aho-Corasick](https://en.wikipedia.org/wiki/Aho%E2%80%93Corasick_algorithm) algorithm. There are both Rust and Python implementations.
+Main difference form Aho-Corasick is that BlitzText only matches the longest pattern in a greedy manner.
+
+<img src="benches/benchmark_results_single.png">
+
+## Table of Contents
+- [Installation](#installation)
+- [Usage](#usage)
+    - [Rust Usage](#rust-usage)
+    - [Python Usage](#python-usage)
+- [Features](#Features)
+    - [1. Parallel Processing](#1-parallel-processing)
+    - [2. Fuzzy Matching](#2-fuzzy-matching)
+    - [3. Case Sensitivity](#3-case-sensitivity)
+    - [4. Overlapping Matches](#4-overlapping-matches)
+    - [5. Custom Non-Word Boundaries](#5-custom-non-word-boundaries)
+- [Performance](#performance)
+- [Contributing](#contributing)
+- [License](#license)
+
+
+## Installation
+
+### Rust
+
+Add this to your `Cargo.toml`:
+
+```toml
+[dependencies]
+blitztext = "0.1.0"
+```
+
+or
+
+```shell
+cargo add blitztext
+```
+
+
+### Python
+
+Install the library using pip:
+
+```
+pip install blitztext
+```
+
+## Usage
+
+### Rust Usage
+
+```rust
+use blitztext::KeywordProcessor;
+
+fn main() {
+    let mut processor = KeywordProcessor::new();
+    processor.add_keyword("rust", Some("Rust Lang"));
+    processor.add_keyword("programming", Some("Coding"));
+
+    let text = "I love rust programming";
+    let matches = processor.extract_keywords(text, None);
+    
+    for m in matches {
+        println!("Found '{}' at [{}, {}]", m.keyword, m.start, m.end);
+    }
+
+    let replaced = processor.replace_keywords(text, None);
+    println!("Replaced text: {}", replaced);
+    // Output: "I love Rust Lang Coding"
+}
+```
+
+### Python Usage
+
+```python
+from blitztext import KeywordProcessor
+
+processor = KeywordProcessor()
+processor.add_keyword("rust", "Rust Lang")
+processor.add_keyword("programming", "Coding")
+
+text = "I love rust programming"
+matches = processor.extract_keywords(text)
+
+for m in matches:
+    print(f"Found '{m.keyword}' at [{m.start}, {m.end}]")
+
+replaced = processor.replace_keywords(text)
+// Output: "I love Rust Lang Coding"
+
+print(f"Replaced text: {replaced}")
+```
+
+## Features
+
+### 1. Parallel Processing
+
+For processing multiple texts in parallel:
+
+```rust
+// Rust
+let texts = vec!["Text 1", "Text 2", "Text 3"];
+let results = processor.parallel_extract_keywords_from_texts(&texts, None);
+```
+
+```python
+# Python
+texts = ["Text 1", "Text 2", "Text 3"]
+results = processor.parallel_extract_keywords_from_texts(texts)
+```
+
+
+### 2. Fuzzy Matching
+
+Both Rust and Python implementations support fuzzy matching:
+
+```rust
+// Rust
+let matches = processor.extract_keywords(text, Some(0.8));
+```
+
+```python
+# Python
+matches = processor.extract_keywords(text, threshold=0.8)
+```
+
+### 3. Case Sensitivity
+
+You can enable case-sensitive matching:
+
+```rust
+// Rust
+let mut processor = KeywordProcessor::with_options(true, false);
+processor.add_keyword("Rust", Some("Rust Lang"));
+let matches = processor.extract_keywords("I love Rust and rust", None);
+// Only "Rust" will be matched, not "rust"
+```
+
+```python
+# Python
+processor = KeywordProcessor(case_sensitive=True)
+processor.add_keyword("Rust", "Rust Lang")
+matches = processor.extract_keywords("I love Rust and rust")
+# Only "Rust" will be matched, not "rust"
+```
+
+### 4. Overlapping Matches
+
+Enable overlapping matches:
+
+```rust
+// Rust
+let mut processor = KeywordProcessor::with_options(false, true);
+processor.add_keyword("word", None);
+processor.add_keyword("sword", None);
+let matches = processor.extract_keywords("I have a sword", None);
+// "word" will be matched
+```
+
+```python
+# Python
+processor = KeywordProcessor(allow_overlaps=True)
+processor.add_keyword("word")
+matches = processor.extract_keywords("I have a sword")
+# "word" will be matched
+```
+
+
+### 5. Custom Non-Word Boundaries
+
+This library uses the concept of non-word boundaries to determine where words begin and end. By default, alphanumeric characters and underscores are considered part of a word. You can customize this behavior to fit your specific needs.
+
+#### Understanding Non-Word Boundaries
+
+- Characters defined as non-word boundaries are considered part of a word.
+- Characters not defined as non-word boundaries are treated as word separators.
+
+#### Example
+
+```rust
+// Rust
+let mut processor = KeywordProcessor::new();
+
+processor.add_keyword("rust", None);
+processor.add_keyword("programming", Some("coding"));
+
+let text = "I-love-rust-programming-and-1coding2";
+
+// Default behavior: '-' is a word separator
+let matches = processor.extract_keywords(text, None);
+assert_eq!(matches.len(), 2);
+// Matches: "rust" and "coding"
+
+// Add '-' as a non-word boundary
+processor.add_non_word_boundary('-');
+
+// Now '-' is considered part of words
+let matches = processor.extract_keywords(text, None);
+assert_eq!(matches.len(), 0);
+// No matches, because "rust" and "programming" are now part of larger "words"
+```
+
+```python
+# Python
+processor = KeywordProcessor()
+
+processor.add_keyword("rust")
+processor.add_keyword("programming", "coding")
+
+text = "I-love-rust-programming-and-1coding2"
+
+# Default behavior: '-' is a word separator
+matches = processor.extract_keywords(text)
+assert len(matches) == 2
+# Matches: "rust" and "coding"
+
+# Add '-' as a non-word boundary
+processor.add_non_word_boundary('-')
+
+# Now '-' is considered part of words
+matches = processor.extract_keywords(text)
+assert len(matches) == 0
+# No matches, because "rust" and "programming" are now part of larger "words"
+```
+
+#### Setting a whole new set of non-word boundaries
+
+```rust
+// Rust
+processor.set_non_word_boundaries(&['-', '_', '@']);
+```
+
+```python
+# Python
+processor.set_non_word_boundaries(['-', '_', '@'])
+```
+
+
+## Performance
+
+---
+
+BlitzText is designed for high performance, making it suitable for processing large volumes of text.
+Benchmark details [here](benches/benchmark_results.md).
+
+Mult-threaded performance:
+<img src="benches/benchmark_results_multi.png">
+
+## Contributing
+
+---
+
+Contributions are welcome! Please feel free to submit a Pull Request.
+
+## Issues
+If you encounter any problems, please file an [issue](https://github.com/praise2112/blitztext/issues) along with a detailed description.
+
+## License
+
+---
+
+This project is licensed under the [MIT License](https://github.com/praise2112/blitztext/blob/main/LICENSE).
+
+

blitztext-0.1.0.dist-info/RECORD

@@ -0,0 +1,8 @@
+blitztext-0.1.0.dist-info/METADATA,sha256=Q7EzYeFF_Xt8cI6rJPIIusW0VAh07OVRxwXcYAzxTLo,7388
+blitztext-0.1.0.dist-info/WHEEL,sha256=efqlbyoWVaRpv3_pmcpfQMmn5BxUR3OkPLmrUDmnQP0,90
+blitztext-0.1.0.dist-info/license_files/LICENSE,sha256=M96WJMlfMKb2MSk9-Wnb432DqTdlu9qQiuLS0vutwEY,1119
+blitztext/__init__.py,sha256=pOLO1hwZxoTihScRSVIFZYggcF89bbSUaeAETVbzv5E,119
+blitztext/__init__.pyi,sha256=edWR9hXhqqGt3RDrMYevMT5rziujUWsq8I8fmK-blyk,4970
+blitztext/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+blitztext/blitztext.cp38-win32.pyd,sha256=ZEpE-3Kg76vw7uMBiyGn37gjFj00sh3w79FLoBoL2gU,399360
+blitztext-0.1.0.dist-info/RECORD,,

blitztext-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: maturin (1.7.0)
+Root-Is-Purelib: false
+Tag: cp38-none-win32

blitztext-0.1.0.dist-info/license_files/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Praise Oketola (oketola.praise@gmail.com)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.