dalla-data-processing 0.0.10__tar.gz → 0.0.11__tar.gz

{dalla_data_processing-0.0.10 → dalla_data_processing-0.0.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dalla-data-processing
-Version: 0.0.10
+Version: 0.0.11
 Summary: data processing pipeline with deduplication, stemming, quality checking, and readability scoring, used for the DALLA Models
 Author-email: Hadi Hamoud <hhamoud@dohainstitute.edu.qa>, Digital Research Unit - Arab Center <dru@dohainstitute.edu.qa>
 License: CC-BY-NC-SA-4.0

{dalla_data_processing-0.0.10 → dalla_data_processing-0.0.11}/dalla_data_processing/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.0.10'
-__version_tuple__ = version_tuple = (0, 0, 10)
+__version__ = version = '0.0.11'
+__version_tuple__ = version_tuple = (0, 0, 11)
 
-__commit_id__ = commit_id = 'gcc87ead80'
+__commit_id__ = commit_id = 'g5e00041e7'

{dalla_data_processing-0.0.10 → dalla_data_processing-0.0.11}/dalla_data_processing/utils/__init__.py

@@ -1,8 +1,4 @@
-"""
-Utility functions for text processing.
-
-This module provides utilities for tokenization, text manipulation, and logging.
-"""
+"""Utility functions for text processing."""
 
 from dalla_data_processing.utils.logger import get_logger, logger, setup_logging
 

dalla_data_processing-0.0.11/dalla_data_processing/utils/tokenize.py

@@ -0,0 +1,80 @@
+# MIT License
+#
+# Copyright 2018-2024 New York University Abu Dhabi
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+"""Word-boundary tokenization utilities."""
+
+import re
+
+__all__ = ["simple_word_tokenize"]
+
+# Compact mode: Arabic + Latin + digits
+_ARABIC = (
+    r"\u0621-\u063A"
+    r"\u0641-\u064A"
+    r"\u064B-\u0652"
+    r"\u0653-\u0655"
+    r"\u0670"
+    r"\u0671-\u06D3"
+    r"\u06D5-\u06FF"
+    r"\u0750-\u077F"
+    r"\u08A0-\u08FF"
+    r"\uFB50-\uFDFF"
+    r"\uFE70-\uFEFF"
+)
+_LATIN = r"a-zA-Z"
+_DIGITS = r"0-9\u0660-\u0669\u06F0-\u06F9"
+_COMPACT_CHARSET = _ARABIC + _LATIN + _DIGITS
+
+# Full mode: Unicode letters/marks/numbers (via \w which covers all Unicode word chars)
+_FULL_CHARSET = r"\w"
+
+# Pre-compiled regexes for compact mode
+_COMPACT_RE = re.compile(f"[{_COMPACT_CHARSET}]+|[^{_COMPACT_CHARSET}\\s]|\\s+")
+_COMPACT_SPLIT_RE = re.compile(f"[{_ARABIC}{_LATIN}]+|[{_DIGITS}]+|[^{_COMPACT_CHARSET}\\s]|\\s+")
+
+# Pre-compiled regexes for full mode
+_FULL_RE = re.compile(r"\w+|[^\w\s]|\s+")
+_FULL_SPLIT_RE = re.compile(r"[^\W\d]+|\d+|[^\w\s]|\s+")
+
+
+def simple_word_tokenize(sentence, split_digits=False, mode="compact"):
+    """Tokenize a sentence by splitting on whitespace and separating punctuation.
+
+    Args:
+        sentence: Sentence to tokenize.
+        split_digits: Split digits from letters. Defaults to False.
+        mode: "compact" (Arabic + Latin + digits) or "full" (all Unicode).
+            Defaults to "compact".
+
+    Returns:
+        List of tokens.
+    """
+    if mode == "compact":
+        if split_digits:
+            return _COMPACT_SPLIT_RE.findall(sentence)
+        return _COMPACT_RE.findall(sentence)
+    elif mode == "full":
+        if split_digits:
+            return _FULL_SPLIT_RE.findall(sentence)
+        return _FULL_RE.findall(sentence)
+    else:
+        raise ValueError(f"Unknown mode: {mode}. Use 'compact' or 'full'.")

{dalla_data_processing-0.0.10 → dalla_data_processing-0.0.11}/dalla_data_processing.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dalla-data-processing
-Version: 0.0.10
+Version: 0.0.11
 Summary: data processing pipeline with deduplication, stemming, quality checking, and readability scoring, used for the DALLA Models
 Author-email: Hadi Hamoud <hhamoud@dohainstitute.edu.qa>, Digital Research Unit - Arab Center <dru@dohainstitute.edu.qa>
 License: CC-BY-NC-SA-4.0

dalla_data_processing-0.0.10/dalla_data_processing/utils/tokenize.py

@@ -1,89 +0,0 @@
-# MIT License
-#
-# Copyright 2018-2024 New York University Abu Dhabi
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-# SOFTWARE.
-
-
-"""This module contains utilities for word-boundary tokenization."""
-
-import re
-
-from camel_tools.utils.charsets import (
-    EMOJI_MULTICHAR_CHARSET,
-    UNICODE_LETTER_CHARSET,
-    UNICODE_LETTER_MARK_NUMBER_CHARSET,
-    UNICODE_MARK_CHARSET,
-    UNICODE_NUMBER_CHARSET,
-    UNICODE_PUNCT_SYMBOL_CHARSET,
-)
-
-__all__ = ["simple_word_tokenize"]
-
-
-_ALL_PUNCT_SYMBOLS = UNICODE_PUNCT_SYMBOL_CHARSET | EMOJI_MULTICHAR_CHARSET
-_ALL_PUNCT_SYMBOLS = [re.escape(x) for x in _ALL_PUNCT_SYMBOLS]
-_ALL_PUNCT_SYMBOLS = sorted(_ALL_PUNCT_SYMBOLS, key=len, reverse=True)
-_WHITESPACE_RE = r"\s+"
-_ALL_NUMBER = "".join(UNICODE_NUMBER_CHARSET)
-_ALL_LETTER_MARK = "".join(UNICODE_LETTER_CHARSET | UNICODE_MARK_CHARSET)
-_ALL_LETTER_MARK_NUMBER = "".join(UNICODE_LETTER_MARK_NUMBER_CHARSET)
-
-_TOKENIZE_RE = re.compile(
-    "|".join(_ALL_PUNCT_SYMBOLS)
-    + r"|["
-    + re.escape(_ALL_LETTER_MARK_NUMBER)
-    + r"]+|"
-    + _WHITESPACE_RE
-)
-_TOKENIZE_NUMBER_RE = re.compile(
-    "|".join(_ALL_PUNCT_SYMBOLS)
-    + r"|["
-    + re.escape(_ALL_NUMBER)
-    + r"]+|["
-    + re.escape(_ALL_LETTER_MARK)
-    + r"]+"
-)
-
-
-def simple_word_tokenize(sentence, split_digits=False):
-    """Tokenizes a sentence by splitting on whitespace and seperating
-    punctuation. The resulting tokens are either alpha-numeric words, single
-    punctuation/symbol/emoji characters, or multi-character emoji sequences.
-    This function is language agnostic and splits all characters marked as
-    punctuation or symbols in the Unicode specification.
-    For example, tokenizing :code:`'Hello,    world!!!'`
-    would yield :code:`['Hello', ',', 'world', '!', '!', '!']`.
-    If split_digits is set to True, it also splits on number.
-    For example, tokenizing :code:`'Hello,    world123!!!'`
-    would yield :code:`['Hello', ',', 'world', '123', '!', '!', '!']`.
-
-    Args:
-        sentence (:obj:`str`): Sentence to tokenize.
-        split_digits (:obj:`bool`, optional): The flag to split on number.
-            Defaults to False.
-
-    Returns:
-        :obj:`list` of :obj:`str`: The list of tokens.
-    """
-
-    if split_digits:
-        return _TOKENIZE_NUMBER_RE.findall(sentence)
-    else:
-        return _TOKENIZE_RE.findall(sentence)