ahocorasick-ner 0.0.1__tar.gz

ahocorasick-ner-0.0.1/PKG-INFO

@@ -0,0 +1,13 @@
+Metadata-Version: 2.1
+Name: ahocorasick-ner
+Version: 0.0.1
+Summary: A fast, dictionary-based Named Entity Recognition system using the Aho-Corasick algorithm.
+Home-page: https://github.com/TigreGotico/ahocorasick-ner
+Author: JarbasAI
+Author-email: jarbasai@mailfence.com
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Text Processing :: Linguistic
+Classifier: Intended Audience :: Developers
+Requires-Python: >=3.9

ahocorasick-ner-0.0.1/README.md

@@ -0,0 +1,106 @@
+# AhocorasickNER
+
+A fast and simple Named Entity Recognition (NER) tool based on the Aho-Corasick algorithm. This package is ideal for rule-based entity extraction using pre-defined vocabularies, especially when speed and scalability matter.
+
+---
+
+## ✨ Features
+
+- ✅ Ultra-fast multi-pattern string matching using [Aho-Corasick](https://en.wikipedia.org/wiki/Aho%E2%80%93Corasick_algorithm)
+- ✅ Word-boundary-aware matching
+- ✅ Case-sensitive or case-insensitive modes
+- ✅ Minimal dependencies
+- ✅ Designed for integration with Hugging Face Datasets and similar sources
+
+---
+
+## 🧠 Theoretical Background
+
+The Aho-Corasick algorithm, developed by Alfred V. Aho and Margaret J. Corasick in 1975, constructs a **finite state machine** from a set of keywords to allow **simultaneous pattern matching** in linear time. It's similar to a Trie, but extended with failure transitions that allow it to efficiently handle mismatches and overlapping substrings.
+
+This approach is **ideal for dictionary-based NER systems**, where:
+- You have a fixed list of entities (e.g., names, locations, products)
+- You want to search for **many patterns in a single pass**
+- Speed and low memory usage are critical
+
+Unlike statistical or neural NER systems, this approach doesn't require training and is fully deterministic. It is particularly useful when:
+- You want **consistent results**
+- You are working with **domain-specific vocabularies**
+- You need to process **large corpora quickly**
+
+---
+
+## 🚀 Installation
+
+```bash
+pip install ahocorasick-ner
+```
+
+---
+
+## 🛠️ Usage
+
+```python
+from ahocorasick_ner import AhocorasickNER
+from datasets import load_dataset
+
+EncyclopediaMetallvm = AhocorasickNER()
+
+dataset_name = "Jarbas/metal-archives-tracks"
+dataset = load_dataset(dataset_name)["train"]
+for entry in dataset:
+    EncyclopediaMetallvm.add_word("artist_name", entry["band_name"])
+    if entry.get("track_name"):
+        EncyclopediaMetallvm.add_word("track_name", entry["track_name"])
+    if entry.get("album_name"):
+        EncyclopediaMetallvm.add_word("album_name", entry["album_name"])
+    EncyclopediaMetallvm.add_word("album_type", entry["album_type"])
+
+dataset_name = "Jarbas/metal-archives-bands"
+dataset = load_dataset(dataset_name)["train"]
+for entry in dataset:
+    EncyclopediaMetallvm.add_word("artist_name", entry["name"])
+    if entry.get("genre"):
+        EncyclopediaMetallvm.add_word("music_genre", entry["genre"])
+    if entry.get("label"):
+        EncyclopediaMetallvm.add_word("record_label", entry["label"])
+    if entry.get("country"):
+        EncyclopediaMetallvm.add_word("country", entry["country"])
+
+for entity in EncyclopediaMetallvm.tag("I fucking love black metal from Norway"):
+    print(entity)
+```
+
+### Output:
+```python
+{'start': 15, 'end': 25, 'word': 'black metal', 'label': 'genre'}
+{'start': 32, 'end': 37, 'word': 'Norway', 'label': 'country'}
+```
+
+---
+
+## 🧪 Benchmarks
+
+With 100k+ known phrases, this tool can tag documents in milliseconds thanks to the Aho-Corasick FSM structure. It scales gracefully with both the number of patterns and the size of input text.
+
+---
+
+## 🧩 Limitations
+
+- Does not handle nested or overlapping entities well (greedy, longest match wins)
+- No fuzzy matching (e.g., typos or misspellings won't match)
+- Requires all entities to be known beforehand
+
+
+---
+
+## 📄 License
+
+MIT — free for commercial and non-commercial use.
+
+---
+
+## 🙏 Acknowledgements
+
+- [pyahocorasick](https://github.com/WojciechMula/pyahocorasick) — The underlying C-based Aho-Corasick implementation
+- [Hugging Face Datasets](https://huggingface.co/docs/datasets) — For loading domain-specific corpora

ahocorasick-ner-0.0.1/ahocorasick_ner/__init__.py

@@ -0,0 +1,99 @@
+import ahocorasick
+import pickle
+import re
+from typing import Dict, Iterable, List, Tuple, Set
+
+
+class AhocorasickNER:
+    """
+    A simple Named Entity Recognition system using the Aho-Corasick algorithm.
+    Supports matching pre-defined entities in a given string with word boundary filtering.
+    """
+
+    def __init__(self, case_sensitive: bool = False):
+        """
+        Initialize the NER system.
+
+        Args:
+            case_sensitive (bool): Whether matching should be case-sensitive. Defaults to False.
+        """
+        self.automaton = ahocorasick.Automaton()
+        self.case_sensitive = case_sensitive
+        self._fitted = False
+
+    def save(self, path: str):
+        self.automaton.save(path, pickle.dumps)
+
+    def load(self, path: str):
+        self.automaton = ahocorasick.load(path, pickle.loads)
+
+    def add_word(self, label: str, example: str) -> None:
+        """
+        Add a labeled word or phrase to the automaton.
+
+        Args:
+            label (str): The label to associate with the word (e.g., 'artist_name').
+            example (str): The word or phrase to match.
+        """
+        key = example if self.case_sensitive else example.lower()
+        self.automaton.add_word(key, (label, key))
+        self._fitted = False
+
+    def fit(self) -> None:
+        """
+        Finalize the automaton. This must be called after all words are added.
+        """
+        if not self._fitted:
+            self.automaton.make_automaton()
+        self._fitted = True
+
+    def tag(self, haystack: str, min_word_len: int = 5) -> Iterable[Dict[str, str]]:
+        """
+        Search for labeled entities in the given string.
+
+        Args:
+            haystack (str): The input string to search.
+            min_word_len (int): Minimum word length to consider a match. Defaults to 5.
+
+        Yields:
+            Dict[str, str]: A dictionary with keys 'start', 'end', 'word', and 'label'.
+        """
+        if not self._fitted:
+            self.fit()
+
+        processed_haystack = haystack if self.case_sensitive else haystack.lower()
+        matches: List[Tuple[int, int, str, str]] = []
+
+        for idx, (label, word) in self.automaton.iter(processed_haystack):
+            if len(word) < min_word_len:
+                continue
+
+            start = idx - len(word) + 1
+            end = idx
+
+            # Respect word boundaries
+            before = processed_haystack[start - 1] if start > 0 else ' '
+            after = processed_haystack[end + 1] if end + 1 < len(processed_haystack) else ' '
+            if re.match(r'\w', before) or re.match(r'\w', after):
+                continue  # skip partial word matches
+
+            matches.append((start, end, word, label))
+
+        # Sort by descending length, then by start position
+        matches.sort(key=lambda x: (-(x[1] - x[0] + 1), x[0]))
+
+        selected: List[Tuple[int, int, str, str]] = []
+        used: Set[int] = set()
+
+        for start, end, word, label in matches:
+            if all(i not in used for i in range(start, end + 1)):
+                selected.append((start, end, word, label))
+                used.update(range(start, end + 1))
+
+        for start, end, word, label in sorted(selected, key=lambda x: x[0]):
+            yield {
+                "start": start,
+                "end": end,
+                "word": haystack[start:end + 1],
+                "label": label
+            }

ahocorasick-ner-0.0.1/ahocorasick_ner/datasets.py

@@ -0,0 +1,180 @@
+import ahocorasick
+import os
+from ahocorasick_ner import AhocorasickNER
+
+try:
+    from datasets import load_dataset
+except ImportError as e:
+    # only used in demo classes, not a hard requirement
+    def load_dataset(*args, **kwargs):
+        raise e
+
+
+class EncyclopediaMetallvmNER(AhocorasickNER):
+    def __init__(self, path: str | None = None, case_sensitive: bool = False):
+        super().__init__(case_sensitive)
+        if path and os.path.exists(path):
+            self.load(path)
+        else:
+            self.train(path)
+
+    def train(self, path: str | None = None):
+        self.load_huggingface()
+        if path:
+            self.save(path)
+
+    def load_huggingface(self):
+        dataset_name = "Jarbas/metal-archives-tracks"
+        dataset = load_dataset(dataset_name)["train"]
+        for entry in dataset:
+            self.add_word("artist_name", entry["band_name"])
+            if entry.get("track_name"):
+                self.add_word("track_name", entry["track_name"])
+            if entry.get("album_name"):
+                self.add_word("album_name", entry["album_name"])
+            self.add_word("album_type", entry["album_type"])
+
+        dataset_name = "Jarbas/metal-archives-bands"
+        dataset = load_dataset(dataset_name)["train"]
+        for entry in dataset:
+            self.add_word("artist_name", entry["name"])
+            if entry.get("genre"):
+                self.add_word("music_genre", entry["genre"])
+            if entry.get("label"):
+                self.add_word("record_label", entry["label"])
+
+
+class MusicNER(AhocorasickNER):
+    def __init__(self, path: str | None = None,
+                 case_sensitive: bool = False):
+        super().__init__(case_sensitive)
+        if path and os.path.exists(path):
+            self.load(path)
+        else:
+            self.train(path)
+
+    def train(self, path: str | None = None):
+        self.load_huggingface()
+        if path:
+            self.save(path)
+
+    def load_huggingface(self):
+        self.load_metallvm()
+        self.load_jazz()
+        self.load_prog()
+        self.load_classical()
+        self.load_trance()
+
+    def load_trance(self):
+        dataset_name = "Jarbas/trance_tracks"
+        dataset = load_dataset(dataset_name)["train"]
+        for entry in dataset:
+            if entry.get("ARTIST(S)"):
+                self.add_word("artist_name", entry["ARTIST(S)"])
+            if entry.get("TRACK"):
+                self.add_word("track_name", entry["TRACK"])
+            if entry.get("STYLE"):
+                self.add_word("music_genre", entry["STYLE"])
+
+    def load_classical(self):
+        dataset_name = "Jarbas/classic-composers"
+        dataset = load_dataset(dataset_name)["train"]
+        for entry in dataset:
+            if entry.get("name"):
+                self.add_word("artist_name", entry["name"])
+
+    def load_prog(self):
+        dataset_name = "Jarbas/prog-archives"
+        dataset = load_dataset(dataset_name)["train"]
+        for entry in dataset:
+            if entry.get("artist"):
+                self.add_word("artist_name", entry["artist"])
+            if entry.get("genre"):
+                self.add_word("music_genre", entry["genre"])
+
+    def load_jazz(self):
+        dataset_name = "Jarbas/jazz-music-archives"
+        dataset = load_dataset(dataset_name)["train"]
+        for entry in dataset:
+            if entry.get("artist"):
+                self.add_word("artist_name", entry["artist"])
+            if entry.get("genre"):
+                self.add_word("music_genre", entry["genre"])
+
+    def load_metallvm(self):
+        dataset_name = "Jarbas/metal-archives-tracks"
+        dataset = load_dataset(dataset_name)["train"]
+        for entry in dataset:
+            self.add_word("artist_name", entry["band_name"])
+            if entry.get("track_name"):
+                self.add_word("track_name", entry["track_name"])
+            if entry.get("album_name"):
+                self.add_word("album_name", entry["album_name"])
+            self.add_word("album_type", entry["album_type"])
+
+        dataset_name = "Jarbas/metal-archives-bands"
+        dataset = load_dataset(dataset_name)["train"]
+        for entry in dataset:
+            self.add_word("artist_name", entry["name"])
+            if entry.get("genre"):
+                self.add_word("music_genre", entry["genre"])
+            if entry.get("label"):
+                self.add_word("record_label", entry["label"])
+
+
+class ImdbNER(AhocorasickNER):
+    def __init__(self, path: str | None = None, case_sensitive: bool = False):
+        super().__init__(case_sensitive)
+        if path and os.path.exists(path):
+            self.load(path)
+        else:
+            self.train(path)
+
+    def train(self, path: str | None = None):
+        self.load_huggingface()
+        if path:
+            self.save(path)
+
+    def load_huggingface(self):
+        dataset_name = "Jarbas/movie_actors"
+        dataset = load_dataset(dataset_name)["train"]
+        for entry in dataset:
+            if entry.get("name"):
+                self.add_word("movie_actor", entry["name"])
+
+        dataset_name = "Jarbas/movie_directors"
+        dataset = load_dataset(dataset_name)["train"]
+        for entry in dataset:
+            if entry.get("name"):
+                self.add_word("movie_director", entry["name"])
+
+        dataset_name = "Jarbas/movie_producers"
+        dataset = load_dataset(dataset_name)["train"]
+        for entry in dataset:
+            if entry.get("name"):
+                self.add_word("movie_producer", entry["name"])
+
+        dataset_name = "Jarbas/movie_writers"
+        dataset = load_dataset(dataset_name)["train"]
+        for entry in dataset:
+            if entry.get("name"):
+                self.add_word("movie_writer", entry["name"])
+
+        dataset_name = "Jarbas/movie_composers"
+        dataset = load_dataset(dataset_name)["train"]
+        for entry in dataset:
+            if entry.get("name"):
+                self.add_word("movie_composer", entry["name"])
+
+
+if __name__ == "__main__":
+    import time
+
+    # e = MusicNER("media_net.ahocorasick")
+    # e = ImdbNER("imdb.ahocorasick")
+    e = EncyclopediaMetallvmNER("metallvm.ahocorasick")
+
+    s = time.monotonic()
+    for entity in e.tag("I fucking love black metal"):
+        print(entity)
+    print(time.monotonic() - s)

ahocorasick-ner-0.0.1/ahocorasick_ner/version.py

@@ -0,0 +1,6 @@
+# START_VERSION_BLOCK
+VERSION_MAJOR = 0
+VERSION_MINOR = 0
+VERSION_BUILD = 1
+VERSION_ALPHA = 0
+# END_VERSION_BLOCK

ahocorasick-ner-0.0.1/ahocorasick_ner.egg-info/PKG-INFO

@@ -0,0 +1,13 @@
+Metadata-Version: 2.1
+Name: ahocorasick-ner
+Version: 0.0.1
+Summary: A fast, dictionary-based Named Entity Recognition system using the Aho-Corasick algorithm.
+Home-page: https://github.com/TigreGotico/ahocorasick-ner
+Author: JarbasAI
+Author-email: jarbasai@mailfence.com
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Text Processing :: Linguistic
+Classifier: Intended Audience :: Developers
+Requires-Python: >=3.9

ahocorasick-ner-0.0.1/ahocorasick_ner.egg-info/SOURCES.txt

@@ -0,0 +1,10 @@
+README.md
+setup.py
+ahocorasick_ner/__init__.py
+ahocorasick_ner/datasets.py
+ahocorasick_ner/version.py
+ahocorasick_ner.egg-info/PKG-INFO
+ahocorasick_ner.egg-info/SOURCES.txt
+ahocorasick_ner.egg-info/dependency_links.txt
+ahocorasick_ner.egg-info/requires.txt
+ahocorasick_ner.egg-info/top_level.txt

ahocorasick-ner-0.0.1/ahocorasick_ner.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

ahocorasick-ner-0.0.1/ahocorasick_ner.egg-info/requires.txt

@@ -0,0 +1 @@
+pyahocorasick

ahocorasick-ner-0.0.1/ahocorasick_ner.egg-info/top_level.txt

@@ -0,0 +1 @@
+ahocorasick_ner

ahocorasick-ner-0.0.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

ahocorasick-ner-0.0.1/setup.py

@@ -0,0 +1,60 @@
+import os.path
+
+import os
+from setuptools import setup
+
+BASEDIR = os.path.abspath(os.path.dirname(__file__))
+
+
+def get_version():
+    """ Find the version"""
+    version_file = os.path.join(BASEDIR, 'ahocorasick_ner', 'version.py')
+    major, minor, build, alpha = (None, None, None, None)
+    with open(version_file) as f:
+        for line in f:
+            if 'VERSION_MAJOR' in line:
+                major = line.split('=')[1].strip()
+            elif 'VERSION_MINOR' in line:
+                minor = line.split('=')[1].strip()
+            elif 'VERSION_BUILD' in line:
+                build = line.split('=')[1].strip()
+            elif 'VERSION_ALPHA' in line:
+                alpha = line.split('=')[1].strip()
+
+            if ((major and minor and build and alpha) or
+                    '# END_VERSION_BLOCK' in line):
+                break
+    version = f"{major}.{minor}.{build}"
+    if int(alpha):
+        version += f"a{alpha}"
+    return version
+
+
+def required(requirements_file):
+    """ Read requirements file and remove comments and empty lines. """
+    with open(os.path.join(BASEDIR, requirements_file), 'r') as f:
+        requirements = f.read().splitlines()
+        return [pkg for pkg in requirements
+                if pkg.strip() and not pkg.startswith("#")]
+
+
+setup(
+    name="ahocorasick-ner",
+    version=get_version(),
+    modules=["ahocorasick_ner"],
+    install_requires=required("requirements.txt"),
+    author="JarbasAI",
+    author_email="jarbasai@mailfence.com",
+    description="A fast, dictionary-based Named Entity Recognition system using the Aho-Corasick algorithm.",
+    # long_description=open("README.md").read(),
+    # long_description_content_type="text/markdown",
+    url="https://github.com/TigreGotico/ahocorasick-ner",
+    classifiers=[
+        "Programming Language :: Python :: 3",
+        "License :: OSI Approved :: MIT License",
+        "Operating System :: OS Independent",
+        "Topic :: Text Processing :: Linguistic",
+        "Intended Audience :: Developers",
+    ],
+    python_requires=">=3.9",
+)