dalla-data-processing 0.0.1__py3-none-any.whl

dalla/stemming/data/words_t.txt

@@ -0,0 +1,7 @@
+الأرسطية
+الراشدية
+القاسمية
+اليوسفية
+الكاظمية
+الشاذلية
+الكاريبية

dalla/utils/__init__.py

@@ -0,0 +1,10 @@
+"""
+Utility functions for text processing.
+
+This module provides utilities for tokenization, text manipulation, and logging.
+"""
+
+from dalla.utils.logger import get_logger, logger, setup_logging
+from dalla.utils.tokenize import simple_word_tokenize
+
+__all__ = ["simple_word_tokenize", "logger", "get_logger", "setup_logging"]

dalla/utils/logger.py

@@ -0,0 +1,128 @@
+"""
+Structured logging configuration using structlog.
+
+This module provides a centralized logging setup for the application,
+with support for both JSON and console output formats.
+"""
+
+import logging
+import sys
+
+import structlog
+from structlog.types import EventDict, Processor
+
+
+def rename_event_key(_, __, event_dict: EventDict) -> EventDict:
+    """
+    Log entries keep the text message in the `event` field, but some systems
+    use the `message` field. This processor moves the value from one field to
+    the other.
+    See https://github.com/hynek/structlog/issues/35#issuecomment-591321744
+    """
+    event_dict["message"] = event_dict.pop("event")
+    return event_dict
+
+
+def drop_color_message_key(_, __, event_dict: EventDict) -> EventDict:
+    """
+    Drop the `color_message` key from the event dict if it exists.
+    Some loggers add this redundant field which we don't need.
+    """
+    event_dict.pop("color_message", None)
+    return event_dict
+
+
+def setup_logging(log_format: str = "console", log_level: str = "INFO"):
+    """
+    Configure structured logging for the application.
+
+    Args:
+        log_format: Output format - either "json" or "console" (default: "console")
+        log_level: Minimum log level - DEBUG, INFO, WARNING, ERROR, or CRITICAL (default: "INFO")
+    """
+    timestamper = structlog.processors.TimeStamper(fmt="iso")
+
+    shared_processors: list[Processor] = [
+        structlog.contextvars.merge_contextvars,
+        structlog.stdlib.add_logger_name,
+        structlog.stdlib.add_log_level,
+        structlog.stdlib.PositionalArgumentsFormatter(),
+        structlog.stdlib.ExtraAdder(),
+        drop_color_message_key,
+        timestamper,
+        structlog.processors.StackInfoRenderer(),
+    ]
+
+    if log_format == "json":
+        # We rename the `event` key to `message` only in JSON logs
+        # The pretty ConsoleRenderer looks for `event`
+        shared_processors.append(rename_event_key)
+        # Format the exception only for JSON logs, as we want to pretty-print them when
+        # using the ConsoleRenderer
+        shared_processors.append(structlog.processors.format_exc_info)
+
+    structlog.configure(
+        processors=shared_processors
+        + [
+            # Prepare event dict for `ProcessorFormatter`.
+            structlog.stdlib.ProcessorFormatter.wrap_for_formatter,
+        ],
+        logger_factory=structlog.stdlib.LoggerFactory(),
+        cache_logger_on_first_use=True,
+    )
+
+    log_renderer: structlog.types.Processor
+    if log_format == "json":
+        log_renderer = structlog.processors.JSONRenderer()
+    else:
+        log_renderer = structlog.dev.ConsoleRenderer()
+
+    formatter = structlog.stdlib.ProcessorFormatter(
+        # These run ONLY on `logging` entries that do NOT originate within
+        # structlog.
+        foreign_pre_chain=shared_processors,
+        # These run on ALL entries after the pre_chain is done.
+        processors=[
+            # Remove _record & _from_structlog.
+            structlog.stdlib.ProcessorFormatter.remove_processors_meta,
+            log_renderer,
+        ],
+    )
+
+    handler = logging.StreamHandler()
+    # Use OUR `ProcessorFormatter` to format all `logging` entries.
+    handler.setFormatter(formatter)
+    root_logger = logging.getLogger()
+    root_logger.addHandler(handler)
+    root_logger.setLevel(log_level.upper())
+
+    def handle_exception(exc_type, exc_value, exc_traceback):
+        """
+        Log any uncaught exception instead of letting it be printed by Python
+        (but leave KeyboardInterrupt untouched to allow users to Ctrl+C to stop)
+        See https://stackoverflow.com/a/16993115/3641865
+        """
+        if issubclass(exc_type, KeyboardInterrupt):
+            sys.__excepthook__(exc_type, exc_value, exc_traceback)
+            return
+
+        root_logger.error("Uncaught exception", exc_info=(exc_type, exc_value, exc_traceback))
+
+    sys.excepthook = handle_exception
+
+
+def get_logger(name: str = "dalla") -> structlog.stdlib.BoundLogger:
+    """
+    Get a structured logger instance.
+
+    Args:
+        name: Logger name (default: "dalla")
+
+    Returns:
+        A configured structlog logger instance
+    """
+    return structlog.stdlib.get_logger(name)
+
+
+# Default logger instance for the application
+logger = get_logger("dalla")

dalla/utils/tokenize.py

@@ -0,0 +1,89 @@
+# MIT License
+#
+# Copyright 2018-2024 New York University Abu Dhabi
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+
+"""This module contains utilities for word-boundary tokenization."""
+
+import re
+
+from camel_tools.utils.charsets import (
+    EMOJI_MULTICHAR_CHARSET,
+    UNICODE_LETTER_CHARSET,
+    UNICODE_LETTER_MARK_NUMBER_CHARSET,
+    UNICODE_MARK_CHARSET,
+    UNICODE_NUMBER_CHARSET,
+    UNICODE_PUNCT_SYMBOL_CHARSET,
+)
+
+__all__ = ["simple_word_tokenize"]
+
+
+_ALL_PUNCT_SYMBOLS = UNICODE_PUNCT_SYMBOL_CHARSET | EMOJI_MULTICHAR_CHARSET
+_ALL_PUNCT_SYMBOLS = [re.escape(x) for x in _ALL_PUNCT_SYMBOLS]
+_ALL_PUNCT_SYMBOLS = sorted(_ALL_PUNCT_SYMBOLS, key=len, reverse=True)
+_WHITESPACE_RE = r"\s+"
+_ALL_NUMBER = "".join(UNICODE_NUMBER_CHARSET)
+_ALL_LETTER_MARK = "".join(UNICODE_LETTER_CHARSET | UNICODE_MARK_CHARSET)
+_ALL_LETTER_MARK_NUMBER = "".join(UNICODE_LETTER_MARK_NUMBER_CHARSET)
+
+_TOKENIZE_RE = re.compile(
+    "|".join(_ALL_PUNCT_SYMBOLS)
+    + r"|["
+    + re.escape(_ALL_LETTER_MARK_NUMBER)
+    + r"]+|"
+    + _WHITESPACE_RE
+)
+_TOKENIZE_NUMBER_RE = re.compile(
+    "|".join(_ALL_PUNCT_SYMBOLS)
+    + r"|["
+    + re.escape(_ALL_NUMBER)
+    + r"]+|["
+    + re.escape(_ALL_LETTER_MARK)
+    + r"]+"
+)
+
+
+def simple_word_tokenize(sentence, split_digits=False):
+    """Tokenizes a sentence by splitting on whitespace and seperating
+    punctuation. The resulting tokens are either alpha-numeric words, single
+    punctuation/symbol/emoji characters, or multi-character emoji sequences.
+    This function is language agnostic and splits all characters marked as
+    punctuation or symbols in the Unicode specification.
+    For example, tokenizing :code:`'Hello,    world!!!'`
+    would yield :code:`['Hello', ',', 'world', '!', '!', '!']`.
+    If split_digits is set to True, it also splits on number.
+    For example, tokenizing :code:`'Hello,    world123!!!'`
+    would yield :code:`['Hello', ',', 'world', '123', '!', '!', '!']`.
+
+    Args:
+        sentence (:obj:`str`): Sentence to tokenize.
+        split_digits (:obj:`bool`, optional): The flag to split on number.
+            Defaults to False.
+
+    Returns:
+        :obj:`list` of :obj:`str`: The list of tokens.
+    """
+
+    if split_digits:
+        return _TOKENIZE_NUMBER_RE.findall(sentence)
+    else:
+        return _TOKENIZE_RE.findall(sentence)

dalla_data_processing-0.0.1.dist-info/METADATA

@@ -0,0 +1,393 @@
+Metadata-Version: 2.4
+Name: dalla-data-processing
+Version: 0.0.1
+Summary: data processing pipeline with deduplication, stemming, quality checking, and readability scoring, used for the DALLA Models
+Author-email: Hadi Hamoud <hhamoud@dohainstitute.edu.qa>, Digital Research Unit - Arab Center <dru@dohainstitute.edu.qa>
+Project-URL: Homepage, https://github.com/U4RASD/dalla-data-processing
+Project-URL: Documentation, https://github.com/U4RASD/dalla-data-processing#readme
+Project-URL: Repository, https://github.com/U4RASD/dalla-data-processing
+Project-URL: Bug Tracker, https://github.com/U4RASD/dalla-data-processing/issues
+Keywords: arabic,nlp,data-processing,deduplication,stemming,readability,quality
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Text Processing :: Linguistic
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+Requires-Dist: datasets>=2.14.0
+Requires-Dist: transformers>=4.30.0
+Requires-Dist: camel-tools>=1.5.0
+Requires-Dist: click>=8.0.0
+Requires-Dist: tqdm>=4.65.0
+Requires-Dist: pandas>=2.0.0
+Requires-Dist: numpy>=1.24.0
+Requires-Dist: pyarrow>=12.0.0
+Requires-Dist: textstat>=0.7.0
+Requires-Dist: structlog>=24.0.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+Requires-Dist: pre-commit>=3.0.0; extra == "dev"
+Provides-Extra: dedup-native
+Requires-Dist: cffi>=1.15.0; extra == "dedup-native"
+Provides-Extra: all
+Requires-Dist: dalla-data-processing[dedup-native,dev]; extra == "all"
+
+# Dalla Data Processing (dalla-dp)
+
+A comprehensive Arabic data processing pipeline with deduplication, stemming, quality checking, and readability scoring, used for the DALLA Models.
+
+## Compatibility
+
+- **Linux**: Fully supported
+- **macOS**: Fully supported (Intel or through rosetta)
+- **Windows**: Supported through WSL (Windows Subsystem for Linux) only, for native windows: manual build from source works for deduplication.
+
+## Installation
+
+<b>Using uv</b>
+
+```bash
+# Install the package
+uv pip install dalla-data-processing
+```
+
+
+<b>Using pip</b>
+
+```bash
+# Install the package
+pip install dalla-data-processing
+```
+
+
+<b>From Source</b>
+
+```bash
+git clone https://github.com/U4RASD/dalla-data-processing.git
+cd dalla-data-processing
+
+# Using uv
+uv pip install -e .
+
+# Or using pip
+pip install -e .
+```
+
+## Components
+
+### Deduplication
+
+**CLI Usage**
+
+**Command:** `dalla-dp deduplicate [OPTIONS]`
+
+**Arguments:**
+- `-t, --threshold FLOAT` - Similarity threshold (0.0-1.0, default: 0.8)
+- `--return-pairs` / `--filter-duplicates` - Return dataset with duplicate info (default) or filtered dataset
+- `--keep-vert-files` - Keep vertical format files for inspection
+- `--vert-dir PATH` - Directory to store vertical files (useful for different disk)
+- `--calculate-scores` - Run phase 2 to calculate similarity scores (slower but more precise)
+- `--onion-binary PATH` - Path to onion binary (auto-detected if not specified)
+
+**Examples:**
+```bash
+# Basic deduplication
+dalla-dp -i ./data/raw -o ./data/deduped deduplicate
+
+# With custom threshold
+dalla-dp -i ./data/raw -o ./data/deduped deduplicate --threshold 0.9
+
+# Return filtered dataset (removes duplicates)
+dalla-dp -i ./data/raw -o ./data/clean deduplicate --filter-duplicates
+
+# Keep intermediate files for inspection
+dalla-dp -i ./data/raw -o ./data/deduped deduplicate --keep-vert-files
+
+# Calculate precise similarity scores (slower)
+dalla-dp -i ./data/raw -o ./data/deduped deduplicate --calculate-scores
+
+# Use custom onion binary
+dalla-dp -i ./data/raw -o ./data/deduped deduplicate --onion-binary /path/to/onion
+```
+
+**Python API**
+
+```python
+from datasets import load_from_disk
+from dalla.deduplication import deduplicate_dataset
+
+# Load dataset
+dataset = load_from_disk("./data/raw")
+
+# get duplicate information (adds columns: duplicate_cluster, is_duplicate, duplicate_count)
+result = deduplicate_dataset(dataset, column="text", threshold=0.8, return_pairs=True)
+
+# filter to see only duplicates
+duplicates = result.filter(lambda x: x['is_duplicate'])
+
+deduped.save_to_disk("./data/clean")
+```
+
+### Stemming
+
+Apply morphological analysis and stemming using CAMeL Tools.
+
+**CLI Usage**
+
+**Command:** `dalla-dp stem [OPTIONS]`
+
+**Arguments:**
+- `--sep-token TEXT` - Separator token for morphological splits (default: `<+>`)
+- `--normalize` - Apply Arabic normalization
+- `--keep-diacritics` - Keep diacritics in output
+- `--model [mle|bert]` - Disambiguator model (default: mle, faster | bert: more accurate)
+- `--use-gpu` - Use GPU for BERT model (only applicable when --model=bert)
+
+**Examples:**
+```bash
+# Basic stemming with MLE model 
+dalla-dp -i ./data/raw -o ./data/stemmed stem
+
+# Use BERT model 
+dalla-dp -i ./data/raw -o ./data/stemmed stem --model bert
+
+# Use BERT with GPU acceleration
+dalla-dp -i ./data/raw -o ./data/stemmed stem --model bert --use-gpu
+
+# Custom separator token
+dalla-dp -i ./data/raw -o ./data/stemmed stem --sep-token "<SEP>"
+
+# Apply normalization
+dalla-dp -i ./data/raw -o ./data/stemmed stem --normalize
+
+# Keep diacritics in output
+dalla-dp -i ./data/raw -o ./data/stemmed stem --keep-diacritics
+
+```
+
+**Python API**
+
+```python
+from datasets import load_from_disk
+from dalla.stemming import stem_dataset
+
+# Load dataset
+dataset = load_from_disk("./data/raw")
+
+stemmed = stem_dataset(dataset, column="text")
+
+stemmed = stem_dataset(
+    dataset,
+    column="text",
+    model="bert",
+    use_gpu=True,
+    num_proc=8
+)
+
+stemmed = stem_dataset(
+    dataset,
+    column="content",
+    sep_token="<+>",
+    normalize=True,
+    keep_diacritics=True
+)
+
+stemmed.save_to_disk("./data/stemmed")
+```
+
+### Quality Checking
+
+Check text quality using morphological analysis to detect errors and foreign words.
+
+**CLI Usage**
+
+**Command:** `dalla-dp quality-check [OPTIONS]`
+
+**Arguments:**
+- `--min-score FLOAT` - Minimum quality score to keep (0-100, default: 0)
+- `--save-errors` - Save erroneous words to file
+- `--model [mle|bert]` - Disambiguator model (default: mle, faster | bert: more accurate)
+- `--use-gpu` - Use GPU for BERT model (only applicable when --model=bert)
+
+**Examples:**
+```bash
+dalla-dp -i ./data/raw -o ./data/quality quality-check
+
+# Filter low-quality texts (score < 50)
+dalla-dp -i ./data/raw -o ./data/quality quality-check --min-score 50
+
+# Save erroneous words to log
+dalla-dp -i ./data/raw -o ./data/quality quality-check --save-errors
+
+# Use BERT model with GPU
+dalla-dp -i ./data/raw -o ./data/quality quality-check --model bert --use-gpu
+
+dalla-dp -i ./data/raw -o ./data/quality -c content quality-check
+```
+
+**Python API**
+
+```python
+from datasets import load_from_disk
+from dalla.quality import check_quality
+
+dataset = load_from_disk("./data/raw")
+
+scored = check_quality(dataset, column="text")
+
+high_quality = check_quality(
+    dataset,
+    column="text",
+    min_score=60.0,
+    save_errors=True
+)
+
+scored = check_quality(
+    dataset,
+    model="bert",
+    use_gpu=True,
+    num_workers=4,
+    timeout=3600
+)
+
+scored.save_to_disk("./data/quality")
+```
+
+### Readability Scoring
+
+Calculate readability scores using Flesch Reading Ease and Osman methods.
+
+**CLI Usage**
+
+**Command:** `dalla-dp readability [OPTIONS]`
+
+**Arguments:**
+- `--add-ranks` / `--no-ranks` - Add ranking and level columns (default: True)
+
+**Examples:**
+```bash
+dalla-dp -i ./data/raw -o ./data/scored readability
+
+dalla-dp -i ./data/raw -o ./data/scored readability --no-ranks
+
+dalla-dp -i ./data/raw -o ./data/scored -c content readability
+```
+
+**Python API**
+
+```python
+from datasets import load_from_disk
+from dalla.readability import score_readability
+
+# Load dataset
+dataset = load_from_disk("./data/raw")
+
+scored = score_readability(dataset, column="text", add_ranks=True)
+
+# Save result
+scored.save_to_disk("./data/scored")
+```
+
+**Readability Levels:**
+- `0`: Very Easy
+- `1`: Easy
+- `2`: Medium
+- `3`: Difficult
+- `4`: Very Difficult
+
+### Dataset Management
+
+Utilities for loading, saving, and inspecting datasets.
+
+**CLI Usage**
+
+**Command:** `dalla-dp info [OPTIONS] DATASET_PATH`
+
+**Arguments:**
+- `DATASET_PATH` - Path to the dataset (required, positional argument)
+- `--split TEXT` - Specific split to show info for
+
+**Examples:**
+```bash
+# Show dataset information
+dalla-dp info ./data/my_dataset
+
+```
+
+**Python API**
+
+```python
+from dalla.core.dataset import DatasetManager
+
+dm = DatasetManager()
+
+dataset = dm.load("./data/my_dataset")
+train_data = dm.load("./data/my_dataset", split="train")
+
+
+info = dm.get_info(dataset)
+dm.print_info(dataset)
+
+size = dm.get_size(dataset)
+
+filtered = dm.filter_dataset(
+    dataset,
+    lambda x: x['quality_score'] > 80.0,
+    num_proc=4
+)
+
+scores = [0.95, 0.87, 0.92, ...]
+dataset = dm.add_column(dataset, "my_score", scores)
+
+subset = dm.select_columns(dataset, ["text", "quality_score"])
+cleaned = dm.remove_columns(dataset, ["temp_column"])
+
+splits = dm.train_test_split(dataset, test_size=0.2, seed=42)
+```
+
+**Working with DatasetDict**
+
+```python
+from datasets import DatasetDict, load_from_disk
+from dalla.quality import check_quality
+
+dataset_dict = load_from_disk("./data/my_dataset")
+
+processed_dict = DatasetDict({
+    split: check_quality(ds, min_score=60.0)
+    for split, ds in dataset_dict.items()
+})
+
+train_processed = check_quality(dataset_dict['train'], min_score=60.0)
+```
+
+## Building Onion from Source
+
+**Build Instructions**
+
+The onion deduplication tool needs to be compiled for your system:
+
+```bash
+cd dalla/deduplication/onion/src_sc
+
+# Compile
+make -f Makefile.g
+
+```
+
+Alternatively, use the build script:
+
+```bash
+chmod +x scripts/build_onion.sh
+./scripts/build_onion.sh
+```
+
+## Links
+
+- Homepage: https://github.com/U4RASD/dalla-data-processing
+- Issues: https://github.com/U4RASD/dalla-data-processing/issues
+- Documentation: https://github.com/U4RASD/dalla-data-processing#readme

dalla_data_processing-0.0.1.dist-info/RECORD

@@ -0,0 +1,55 @@
+dalla/__init__.py,sha256=bkI4Q89iOWvfajFLLN227oILlvLmBs-2XSMe_GU7p9g,574
+dalla/cli.py,sha256=vyTg6QVdTEPgyd10XNj9yHDlO6pWfnlk8QOycoNUjIg,13445
+dalla/core/__init__.py,sha256=Akdu9E15y_w4yVNOiFaNoL4wzozK3bK5DAjRU7Q2Tts,210
+dalla/core/dataset.py,sha256=EVEU-dYu6JNDHEJ3mDQ0Oe2NOts0RJ9jdGqZ7Xfxj0s,12311
+dalla/core/parallel.py,sha256=OjkZu_okQoUmvQTR4_ILcQYXrHlZJdUKIC1LEhB1DZQ,8412
+dalla/deduplication/__init__.py,sha256=2z22nVJ0SiSsoMPRDag5bTUNGnUg9W2daronVyyg-Ds,13191
+dalla/deduplication/onion_wrapper.py,sha256=A_YOc9A_6tyPHro721dqpKkiEJK-INiP9ylM0EhQC9E,6245
+dalla/deduplication/postprocessing.py,sha256=jCOf3-VzcO6sMoPDl7UZOPr1-X2_dWcF317UCFRjJ4E,6276
+dalla/deduplication/preprocessing.py,sha256=-Cf9rj16AcHTA_Xpr-SLnpNmOA-ZDp_tADysO3VcQYU,3071
+dalla/deduplication/bin/.gitignore,sha256=4J55bzduDaecLUxWsg5lYWaYWNdmhJBNQTCNEtYkMFI,8
+dalla/deduplication/bin/onion-linux-x86_64,sha256=JMFQSWLu5c7no41H5aSVZpNoEHC5ZoWI28kZBa5tFGc,116200
+dalla/deduplication/onion/COPYING,sha256=2uaW6t5ZQSZTUzpWPwHk53FcxWx4A4rC5FaPEQejYQg,1428
+dalla/deduplication/onion/Makefile,sha256=cAbImtRPHXmCDlhlm3TJHMDt7_FU5AZrloGGOiNTG18,587
+dalla/deduplication/onion/Makefile.config,sha256=21e5PX_LaLk1Nce0pw4z58BOYF8HiSVCPVBz2Y3F4PM,73
+dalla/deduplication/onion/README.md,sha256=cqoUMWUKQhGG-nnOVS-6cYauA4HzYB4UQanMSavajlI,508
+dalla/deduplication/onion/src/Makefile,sha256=9EsDLZDWHNG0zecIaMZnpW14zGZJcoVT0z3CmVqq8aU,349
+dalla/deduplication/onion/src/Makefile.g,sha256=X_iwBr1UGvr-36fQEnJUuGPq9sBVUI6bpGtAghiJqJs,366
+dalla/deduplication/onion/src/buzhash.c,sha256=UXhOk8t3aE0HqZXNBMF6PgMMb0VkRAMmkeO_R2G9UCY,9281
+dalla/deduplication/onion/src/buzhash.h,sha256=47UdIUVWW0DcBxgxRQsnCpdn6VBYQBelZVPgU0eJ5vk,1206
+dalla/deduplication/onion/src/hashdup.c,sha256=gYylof5UKT0Pd2GUd282OhRV0l9oBMMUnXTcMrpkVnU,4887
+dalla/deduplication/onion/src/hashgen.c,sha256=jHi90u9kJ5x1UbAZ_ksDXlXD1EtarNHvnf1pVsQ0XgY,6494
+dalla/deduplication/onion/src/onion,sha256=01djWtqAWXHu0oBIyuZ-BoxV-qrmasgJYlV2E18KIZE,677200
+dalla/deduplication/onion/src/onion.c,sha256=-8FpfR837Zpx4hmWr6j_QW0z8k_RFHDyLMeGl0tKlLs,26306
+dalla/deduplication/onion/src/onion_dup.c,sha256=tiodAJS50fqarDbN8rNsjFifQm89BwKRfKHuueP-OoA,27581
+dalla/deduplication/onion/src/version.c,sha256=Q2gLN26lVgRNMOsT5fVGa0za3JZO4B2wF2fiCWO7g3g,776
+dalla/deduplication/onion/src/version.h,sha256=ID4eerlhYTU-aDcx_R_eBRotufx1GM-SP5jDOi4Oki0,563
+dalla/deduplication/onion/src_sc/Makefile,sha256=9EsDLZDWHNG0zecIaMZnpW14zGZJcoVT0z3CmVqq8aU,349
+dalla/deduplication/onion/src_sc/Makefile.g,sha256=X_iwBr1UGvr-36fQEnJUuGPq9sBVUI6bpGtAghiJqJs,366
+dalla/deduplication/onion/src_sc/buzhash.c,sha256=UXhOk8t3aE0HqZXNBMF6PgMMb0VkRAMmkeO_R2G9UCY,9281
+dalla/deduplication/onion/src_sc/buzhash.h,sha256=47UdIUVWW0DcBxgxRQsnCpdn6VBYQBelZVPgU0eJ5vk,1206
+dalla/deduplication/onion/src_sc/hashdup,sha256=wWQt0Qn7YrO-6jIV4G0nXlCD2tbyYYuFjU3KypLv5qw,21376
+dalla/deduplication/onion/src_sc/hashdup.c,sha256=gYylof5UKT0Pd2GUd282OhRV0l9oBMMUnXTcMrpkVnU,4887
+dalla/deduplication/onion/src_sc/hashgen,sha256=-zNF_rmTxi-Z9S53wYicAkTToqk8rOcdM3zypXoWd7s,21664
+dalla/deduplication/onion/src_sc/hashgen.c,sha256=jHi90u9kJ5x1UbAZ_ksDXlXD1EtarNHvnf1pVsQ0XgY,6494
+dalla/deduplication/onion/src_sc/onion.c,sha256=1S6DTn2X5a42ZmuBGjvmQyaLQNelr1_QWvD3gMpYx3g,28465
+dalla/deduplication/onion/src_sc/onion_dup.c,sha256=tiodAJS50fqarDbN8rNsjFifQm89BwKRfKHuueP-OoA,27581
+dalla/deduplication/onion/src_sc/version.c,sha256=Q2gLN26lVgRNMOsT5fVGa0za3JZO4B2wF2fiCWO7g3g,776
+dalla/deduplication/onion/src_sc/version.h,sha256=ID4eerlhYTU-aDcx_R_eBRotufx1GM-SP5jDOi4Oki0,563
+dalla/quality/__init__.py,sha256=TAV5rk6UV0sRogoflfuKsaQ4u5opumU1dd7gSIsugRQ,171
+dalla/quality/checker.py,sha256=8n8rLaihhC32gVL4jVj1wH4uVCBM7wPYTrZEVtU--8k,12671
+dalla/readability/__init__.py,sha256=t5dghcv-A5_R7txF9fW9vfdgDc9l_5fFvDwGuJut4OM,6560
+dalla/readability/ranking.py,sha256=dy1UXTHcKmrTwsnT888_-AnoTJTcKjwHBkBhWyfppXw,5382
+dalla/readability/scorer.py,sha256=yFJnH5xXXYo6ga92ceH4z-O01GczrmpB4bIGkIbuMjw,4822
+dalla/stemming/__init__.py,sha256=NbOskEhy4v4GIz2yd7Xcn3gH_YmX_fNaotziqRu8BQ0,19267
+dalla/stemming/data/words_al.txt,sha256=KiekmVjY9B-whpIby7D78C0HSAFy7fxs8RLlOoAVuwc,55876
+dalla/stemming/data/words_al_t.txt,sha256=92WtZbqaM-wCJCKFvckeRxCk6rLJ12ExAnN_NIbDD9M,16047
+dalla/stemming/data/words_t.txt,sha256=V8cpifv1W3XE0lylRkBgRi_Pl8rydIFoB6wVcI25KzA,121
+dalla/utils/__init__.py,sha256=hPQJk-9afcex9zLdfvaTtO_kL-c9mQI-b6S63INYX0Q,326
+dalla/utils/logger.py,sha256=rfnKgwTky5-G_F-GG9l_pZATT-3AMtLSnLU4U43IvDc,4300
+dalla/utils/tokenize.py,sha256=4ZpXqT5faQqh0ySWLDZXr87JUvW8-M-18b6JRfZimZU,3325
+dalla_data_processing-0.0.1.dist-info/METADATA,sha256=W3ZZvo70XF6YhmufQRhbni217r6rNvMMbljBxJMTmxQ,10052
+dalla_data_processing-0.0.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+dalla_data_processing-0.0.1.dist-info/entry_points.txt,sha256=22WRI5lLipfw7ram-Acp_Uk0wlkn8gC0Dy_JTBeCgxw,44
+dalla_data_processing-0.0.1.dist-info/top_level.txt,sha256=eeE-IlXf7Mecax_Sf1XaZmMw1Yx6Ghs3l4l7zAs8YO0,6
+dalla_data_processing-0.0.1.dist-info/RECORD,,

dalla_data_processing-0.0.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

dalla_data_processing-0.0.1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+dalla-dp = dalla.cli:main